[Backport] [2.x] Update bundled JDK to JDK-21.0.1 (#10576) (#11003)

* Update bundled JDK to JDK-21.0.1 (#10576)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
(cherry picked from commit 0d7d1e9)
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Add condition to check JVM version for illegal-access switch (#5279)

This switch is removed in JDK version 17. https://openjdk.org/jeps/403

Signed-off-by: Rabi Panda <adnapibar@gmail.com>

* Check for JDK version while setting java.security.manager option (#5205)

Signed-off-by: Rabi Panda <adnapibar@gmail.com>

* Fix tests and test skaffolding

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

---------

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
Signed-off-by: Rabi Panda <adnapibar@gmail.com>
Co-authored-by: Rabi Panda <adnapibar@gmail.com>

Author: reta

buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java

@@ -111,7 +111,12 @@ public void execute(Task t) {
                         test.systemProperty("java.locale.providers", "SPI,JRE");
                     } else {
                         test.systemProperty("java.locale.providers", "SPI,COMPAT");
-                        test.jvmArgs("--illegal-access=warn");
+                        if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) < 0) {
+                            test.jvmArgs("--illegal-access=warn");
+                        }
+                    }
+                    if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) > 0) {
+                        test.jvmArgs("-Djava.security.manager=allow");
                     }
                 }
             });

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -77,9 +77,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "11.0.20+8";
+    private static final String SYSTEM_JDK_VERSION = "17.0.9+9";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "17.0.8+7";
+    private static final String GRADLE_JDK_VERSION = "17.0.9+9";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

buildSrc/version.properties

@@ -2,7 +2,7 @@ opensearch        = 2.12.0
 lucene            = 9.8.0
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 17.0.8+7
+bundled_jdk = 21.0.1+12
 
 # optional dependencies
 spatial4j         = 0.7

distribution/src/config/jvm.options

@@ -81,7 +81,7 @@ ${error.file}
 
 # JDK 20+ Incubating Vector Module for SIMD optimizations;
 # disabling may reduce performance on vector optimized lucene
-20:--add-modules=jdk.incubator.vector
+20-:--add-modules=jdk.incubator.vector
 
 # HDFS ForkJoinPool.common() support by SecurityManager
 -Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory

distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java

@@ -77,12 +77,21 @@ static List<String> systemJvmOptions() {
                 // log4j 2
                 "-Dlog4j.shutdownHookEnabled=false",
                 "-Dlog4j2.disable.jmx=true",
-
+                // security manager
+                allowSecurityManagerOption(),
                 javaLocaleProviders()
             )
         ).stream().filter(e -> e.isEmpty() == false).collect(Collectors.toList());
     }
 
+    private static String allowSecurityManagerOption() {
+        if (Runtime.version().feature() > 17) {
+            return "-Djava.security.manager=allow";
+        } else {
+            return "";
+        }
+    }
+
     private static String maybeShowCodeDetailsInExceptionMessages() {
         if (Runtime.version().feature() >= 14) {
             return "-XX:+ShowCodeDetailsInExceptionMessages";

libs/core/build.gradle

@@ -104,16 +104,23 @@ tasks.named('forbiddenApisMain').configure {
 tasks.named("thirdPartyAudit").configure {
   ignoreMissingClasses(
     // from log4j
-    'org.osgi.framework.Bundle',
-    'org.osgi.framework.BundleActivator',
-    'org.osgi.framework.BundleContext',
-    'org.osgi.framework.BundleEvent',
-    'org.osgi.framework.FrameworkUtil',
-    'org.osgi.framework.ServiceReference',
-    'org.osgi.framework.ServiceRegistration',
-    'org.osgi.framework.SynchronousBundleListener',
-    'org.osgi.framework.wiring.BundleWire',
-    'org.osgi.framework.wiring.BundleWiring'
+    *[
+        'org.osgi.framework.Bundle',
+        'org.osgi.framework.BundleActivator',
+        'org.osgi.framework.BundleContext',
+        'org.osgi.framework.BundleEvent',
+        'org.osgi.framework.SynchronousBundleListener',
+        'org.osgi.framework.wiring.BundleWire',
+        'org.osgi.framework.wiring.BundleWiring',
+    ]  + (BuildParams.runtimeJavaVersion < JavaVersion.VERSION_20) ? [] : [
+       'jdk.incubator.vector.ByteVector',
+       'jdk.incubator.vector.FloatVector',
+       'jdk.incubator.vector.IntVector',
+       'jdk.incubator.vector.ShortVector',
+       'jdk.incubator.vector.Vector',
+       'jdk.incubator.vector.VectorOperators',
+       'jdk.incubator.vector.VectorSpecies'
+    ]
   )
 }
 

server/build.gradle

@@ -257,6 +257,7 @@ tasks.named("processResources").configure {
 
 tasks.named("thirdPartyAudit").configure {
     ignoreMissingClasses(
+        *[
             // from com.fasterxml.jackson.dataformat.yaml.YAMLMapper (jackson-dataformat-yaml)
             'com.fasterxml.jackson.databind.ObjectMapper',
 
@@ -385,7 +386,17 @@ tasks.named("thirdPartyAudit").configure {
             'org.slf4j.Logger',
             'org.slf4j.LoggerFactory',
             'reactor.blockhound.BlockHound$Builder',
-            'reactor.blockhound.integration.BlockHoundIntegration'
+            'reactor.blockhound.integration.BlockHoundIntegration',
+            'com.google.common.geometry.S2LatLng'
+        ] + (BuildParams.runtimeJavaVersion < JavaVersion.VERSION_20) ? [] : [
+            'jdk.incubator.vector.ByteVector',
+            'jdk.incubator.vector.FloatVector',
+            'jdk.incubator.vector.IntVector',
+            'jdk.incubator.vector.ShortVector',
+            'jdk.incubator.vector.Vector',
+            'jdk.incubator.vector.VectorOperators',
+            'jdk.incubator.vector.VectorSpecies'
+        ]
     )
     ignoreViolations(
             'com.google.protobuf.MessageSchema',

server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java

@@ -58,7 +58,9 @@
 
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.not;
+import static org.junit.Assume.assumeThat;
 
 /**
  * Tests relating to the loss of the cluster-manager.
@@ -71,6 +73,7 @@ public class ClusterManagerDisruptionIT extends AbstractDisruptionTestCase {
      */
     public void testClusterManagerNodeGCs() throws Exception {
         List<String> nodes = startCluster(3);
+        assumeThat("Thread::resume / Thread::suspend are not supported anymore", Runtime.version(), lessThan(Runtime.Version.parse("20")));
 
         String oldClusterManagerNode = internalCluster().getClusterManagerName();
         // a very long GC, but it's OK as we remove the disruption when it has had an effect

server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java

@@ -71,6 +71,8 @@
 
 import static java.util.Collections.singleton;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.lessThan;
+import static org.junit.Assume.assumeThat;
 
 /**
  * Tests relating to the loss of the cluster-manager, but which work with the default fault detection settings which are rather lenient and will
@@ -195,6 +197,8 @@ private void testFollowerCheckerAfterClusterManagerReelection(NetworkLinkDisrupt
      * following another elected cluster-manager node. These nodes should reject this cluster state and prevent them from following the stale cluster-manager.
      */
     public void testStaleClusterManagerNotHijackingMajority() throws Exception {
+        assumeThat("Thread::resume / Thread::suspend are not supported anymore", Runtime.version(), lessThan(Runtime.Version.parse("20")));
+
         final List<String> nodes = internalCluster().startNodes(
             3,
             Settings.builder()

server/src/main/java/org/opensearch/common/util/concurrent/BaseFuture.java

@@ -74,7 +74,7 @@ public abstract class BaseFuture<V> implements Future<V> {
      *
      * @throws InterruptedException  if the current thread was interrupted before
      *                               or during the call (optional but recommended).
-     * @throws CancellationException {@inheritDoc}
+     * @throws CancellationException if the computation was cancelled
      */
     @Override
     public V get(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException, ExecutionException {
@@ -96,7 +96,7 @@ public V get(long timeout, TimeUnit unit) throws InterruptedException, TimeoutEx
      *
      * @throws InterruptedException  if the current thread was interrupted before
      *                               or during the call (optional but recommended).
-     * @throws CancellationException {@inheritDoc}
+     * @throws CancellationException if the computation was cancelled
      */
     @Override
     public V get() throws InterruptedException, ExecutionException {

test/framework/src/main/java/org/opensearch/cluster/coordination/MockSinglePrioritizingExecutor.java

@@ -44,25 +44,30 @@
 public class MockSinglePrioritizingExecutor extends PrioritizedOpenSearchThreadPoolExecutor {
 
     public MockSinglePrioritizingExecutor(String name, DeterministicTaskQueue deterministicTaskQueue, ThreadPool threadPool) {
-        super(name, 0, 1, 0L, TimeUnit.MILLISECONDS, r -> new Thread() {
-            @Override
-            public void start() {
-                deterministicTaskQueue.scheduleNow(new Runnable() {
-                    @Override
-                    public void run() {
-                        try {
-                            r.run();
-                        } catch (KillWorkerError kwe) {
-                            // hacks everywhere
-                        }
+        super(name, 0, 1, 0L, TimeUnit.MILLISECONDS, r -> {
+            // This executor used to override Thread::start method so the actual runnable is
+            // being scheduled in the scope of current thread of execution. In JDK-19, the Thread::start
+            // is not called anymore (https://bugs.openjdk.org/browse/JDK-8292027) and there is no
+            // suitable option to alter the executor's behavior in the similar way. The closest we
+            // could get to is to schedule the runnable once the ThreadFactory is being asked to
+            // allocate the new thread.
+            deterministicTaskQueue.scheduleNow(new Runnable() {
+                @Override
+                public void run() {
+                    try {
+                        r.run();
+                    } catch (KillWorkerError kwe) {
+                        // hacks everywhere
                     }
+                }
 
-                    @Override
-                    public String toString() {
-                        return r.toString();
-                    }
-                });
-            }
+                @Override
+                public String toString() {
+                    return r.toString();
+                }
+            });
+
+            return new Thread(() -> {});
         }, threadPool.getThreadContext(), threadPool.scheduler());
     }
 

test/framework/src/test/java/org/opensearch/test/disruption/LongGCDisruptionTests.java

@@ -48,6 +48,8 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThan;
+import static org.junit.Assume.assumeThat;
 
 public class LongGCDisruptionTests extends OpenSearchTestCase {
 
@@ -65,6 +67,8 @@ public void executeLocked(Runnable r) {
     }
 
     public void testBlockingTimeout() throws Exception {
+        assumeThat("Thread::resume / Thread::suspend are not supported anymore", Runtime.version(), lessThan(Runtime.Version.parse("20")));
+
         final String nodeName = "test_node";
         LongGCDisruption disruption = new LongGCDisruption(random(), nodeName) {
             @Override
@@ -125,6 +129,8 @@ protected long getSuspendingTimeoutInMillis() {
      * but does keep retrying until all threads can be safely paused
      */
     public void testNotBlockingUnsafeStackTraces() throws Exception {
+        assumeThat("Thread::resume / Thread::suspend are not supported anymore", Runtime.version(), lessThan(Runtime.Version.parse("20")));
+
         final String nodeName = "test_node";
         LongGCDisruption disruption = new LongGCDisruption(random(), nodeName) {
             @Override
@@ -179,6 +185,8 @@ protected Pattern[] getUnsafeClasses() {
     }
 
     public void testBlockDetection() throws Exception {
+        assumeThat("Thread::resume / Thread::suspend are not supported anymore", Runtime.version(), lessThan(Runtime.Version.parse("20")));
+
         final String disruptedNodeName = "disrupted_node";
         final String blockedNodeName = "blocked_node";
         CountDownLatch waitForBlockDetectionResult = new CountDownLatch(1);