Merge branch 'main' of github.com:elastic/elasticsearch into custom-inference-service

Author: jonathan-buttner

.backportrc.json

@@ -1,10 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.x", "9.0", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
     "^v9.1.0$" : "main",
-    "^v8.19.0$" : "8.x",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/periodic.template.yml

@@ -237,7 +237,7 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/pipelines/periodic.yml

@@ -656,7 +656,7 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.ci/scripts/resolve-dra-manifest.sh

@@ -24,11 +24,6 @@ if [ "$LATEST_VERSION" != "$ES_VERSION" ]; then
   echo "Latest build for '$ARTIFACT' is version $LATEST_VERSION but expected version $ES_VERSION." 1>&2
   NEW_BRANCH=$(echo $ES_VERSION | sed -E "s/([0-9]+\.[0-9]+)\.[0-9]/\1/g")
 
-  # Temporary
-  if [[ "$ES_VERSION" == "8.16.0" ]]; then
-    NEW_BRANCH="8.x"
-  fi
-
   echo "Using branch $NEW_BRANCH instead of $BRANCH." 1>&2
   LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $NEW_BRANCH)
 fi

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OptimizedScalarQuantizerBenchmark.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.vector;
+
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.index.codec.vectors.OptimizedScalarQuantizer;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(value = 3)
+public class OptimizedScalarQuantizerBenchmark {
+    static {
+        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+    }
+    @Param({ "384", "702", "1024" })
+    int dims;
+
+    float[] vector;
+    float[] centroid;
+    byte[] destination;
+
+    @Param({ "1", "4", "7" })
+    byte bits;
+
+    OptimizedScalarQuantizer osq = new OptimizedScalarQuantizer(VectorSimilarityFunction.DOT_PRODUCT);
+
+    @Setup(Level.Iteration)
+    public void init() {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        // random byte arrays for binary methods
+        destination = new byte[dims];
+        vector = new float[dims];
+        centroid = new float[dims];
+        for (int i = 0; i < dims; ++i) {
+            vector[i] = random.nextFloat();
+            centroid[i] = random.nextFloat();
+        }
+    }
+
+    @Benchmark
+    public byte[] scalar() {
+        osq.scalarQuantize(vector, destination, bits, centroid);
+        return destination;
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public byte[] vector() {
+        osq.scalarQuantize(vector, destination, bits, centroid);
+        return destination;
+    }
+}

branches.json

@@ -8,13 +8,13 @@
       "branch": "9.0"
     },
     {
-      "branch": "8.18"
+      "branch": "8.19"
     },
     {
-      "branch": "8.17"
+      "branch": "8.18"
     },
     {
-      "branch": "8.x"
+      "branch": "8.17"
     },
     {
       "branch": "7.17"

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 9.1.0
-lucene            = 10.2.0
+lucene            = 10.2.1-snapshot-ae6484f43e6
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90
@@ -17,7 +17,6 @@ jna               = 5.12.1
 netty             = 4.1.118.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1
-awsv1sdk            = 1.12.746
 awsv2sdk            = 2.30.38
 reactive_streams    = 1.0.4
 

distribution/docker/build.gradle

@@ -590,7 +590,7 @@ for (final Architecture architecture : Architecture.values()) {
     }
     if(base == DockerBase.DEFAULT) {
       // Add additional docker hub specific context which we use solely for publishing to docker hub.
-      // At the moment it only differs in not labels added that we need for openshift certification
+      // At the moment it is exactly the same as the default context.
       addBuildDockerContextTask(architecture, base, 'DockerHubContext', "docker-hub-build-context")
     }
   }

distribution/docker/src/docker/Dockerfile.default

@@ -139,15 +139,13 @@ LABEL org.label-schema.build-date="${build_date}" \\
   org.opencontainers.image.vendor="Elastic" \\
   org.opencontainers.image.version="${version}"
 
-<% if (docker_context != 'docker-hub-build-context') { %>
 LABEL name="Elasticsearch" \\
   maintainer="infra@elastic.co" \\
   vendor="Elastic" \\
   version="${version}" \\
   release="1" \\
   summary="Elasticsearch" \\
   description="You know, for search."
-<% } %>
 
 RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE
 

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommandTests.java

@@ -193,7 +193,7 @@ public void testStdinInputWithCarriageReturn() throws Exception {
         String password = "keystorepassword";
         KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
-        terminal.addSecretInput("Typedthisandhitenter\r");
+        terminal.addSecretInput("Typedthisandhitenter\r\n");
         execute("-x", "foo");
         assertSecureString("foo", "Typedthisandhitenter", password);
     }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -61,6 +61,13 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Dio.netty.noUnsafe=true",
                 "-Dio.netty.noKeySetOptimization=true",
                 "-Dio.netty.recycler.maxCapacityPerThread=0",
+                // temporary until we get off-heap vector stats in Lucene 10.3
+                "--add-opens=org.apache.lucene.core/org.apache.lucene.codecs.lucene99=org.elasticsearch.server",
+                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene90=org.elasticsearch.server",
+                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene91=org.elasticsearch.server",
+                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene92=org.elasticsearch.server",
+                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene94=org.elasticsearch.server",
+                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene95=org.elasticsearch.server",
                 // log4j 2
                 "-Dlog4j.shutdownHookEnabled=false",
                 "-Dlog4j2.disable.jmx=true",

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java

@@ -38,7 +38,6 @@
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 
-@ESTestCase.WithoutSecurityManager
 public class APMJvmOptionsTests extends ESTestCase {
 
     private Path installDir;

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmErgonomicsTests.java

@@ -15,7 +15,6 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.node.NodeRoleSettings;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -41,7 +40,6 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-@WithoutSecurityManager
 @SuppressFileSystems("*")
 public class JvmErgonomicsTests extends ESTestCase {
 

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java

@@ -15,7 +15,6 @@
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -43,7 +42,6 @@
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.not;
 
-@WithoutSecurityManager
 @LuceneTestCase.SuppressFileSystems("*")
 public class JvmOptionsParserTests extends ESTestCase {
 

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/MachineDependentHeapTests.java

@@ -11,7 +11,6 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
 import org.hamcrest.Matcher;
 
 import java.util.Collections;
@@ -21,7 +20,6 @@
 import static org.hamcrest.Matchers.empty;
 
 // TODO: rework these tests to mock jvm option finder so they can run with security manager, no forking needed
-@WithoutSecurityManager
 public class MachineDependentHeapTests extends ESTestCase {
 
     public void testDefaultHeapSize() throws Exception {

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        10.2.0
-:lucene_version_path:   10_2_0
+:lucene_version:        10.2.1
+:lucene_version_path:   10_2_1
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/125570.yaml

@@ -0,0 +1,5 @@
+pr: 125570
+summary: ES|QL random sampling
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/126035.yaml

@@ -0,0 +1,5 @@
+pr: 126035
+summary: Fix top level knn search with scroll
+area: Vector Search
+type: bug
+issues: []

docs/changelog/126091.yaml

@@ -0,0 +1,5 @@
+pr: 126091
+summary: Allow balancing weights to be set per tier
+area: Allocation
+type: enhancement
+issues: []

docs/changelog/126314.yaml

@@ -0,0 +1,6 @@
+pr: 126314
+summary: Add refresh to synonyms put / delete APIs to wait for synonyms to be accessible and reload analyzers
+area: Analysis
+type: bug
+issues:
+ - 121441

docs/changelog/126397.yaml

@@ -0,0 +1,6 @@
+pr: 126397
+summary: "ESQL: Preserve single aggregate when all attributes are pruned"
+area: ES|QL
+type: bug
+issues:
+ - 126392

docs/changelog/126598.yaml

@@ -0,0 +1,6 @@
+pr: 126598
+summary: "ESQL: Retain aggregate when grouping"
+area: ES|QL
+type: bug
+issues:
+  - 126026

docs/changelog/126629.yaml

@@ -0,0 +1,5 @@
+pr: 126641
+summary: Push more `==`s on text fields to lucene
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/126641.yaml

@@ -0,0 +1,5 @@
+pr: 126653
+summary: Retry shard movements during ESQL query
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/126653.yaml

@@ -0,0 +1,5 @@
+pr: 126704
+summary: Add dense vector off-heap stats to Node stats and Index stats APIs
+area: "Vector Search"
+type: enhancement
+issues: []

docs/changelog/126704.yaml

@@ -0,0 +1,6 @@
+pr: 126783
+summary: Fix shard size of initializing restored shard
+area: Allocation
+type: bug
+issues:
+ - 105331