COMPAT locale provider will be removed in a future release (opensearch-project#14345)

akolarkunnu · harshavamsi · commit 19a569f991ce · 2024-07-12T09:25:19.000-07:00
* COMPAT locale provider will be removed in a future release Description: From JDK21 onwards a new warning has started to come, "WARNING: COMPAT locale provider will be removed in a future release". So, we have to avoid usage of COMPAT provider. We were setting exlpicitly to COMPAT locale provider in couple of places, this change is to convert COMPAT to CLDR locale provider. After this change, couple of tests started to fail becasue some locale data has minor changes in CLDR compared to COMPAT. For example, day and month short names of GERMAN "de" locale are different in CLDR and COMPAT, just need to add a . in the end for CLDR. Resolves opensearch-project#11550 Signed-off-by: Abdul Muneer Kolarkunnu <muneer.kolarkunnu@netapp.com> * COMPAT locale provider will be removed in a future release Description: From JDK21 onwards a new warning has started to come, "WARNING: COMPAT locale provider will be removed in a future release". So, we have to avoid usage of COMPAT provider. We were setting exlpicitly to COMPAT locale provider in couple of places, this change is to convert COMPAT to CLDR locale provider. After this change, couple of tests started to fail becasue some locale data has minor changes in CLDR compared to COMPAT. For example, day and month short names of GERMAN "de" locale are different in CLDR and COMPAT, just need to add a . in the end for CLDR. Resolves opensearch-project#11550 Signed-off-by: Abdul Muneer Kolarkunnu <muneer.kolarkunnu@netapp.com> --------- Signed-off-by: Abdul Muneer Kolarkunnu <muneer.kolarkunnu@netapp.com>
diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md
@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Dependencies
 
 ### Changed
+- Changed locale provider from COMPAT to CLDR  ([#14345](https://github.com/opensearch-project/OpenSearch/pull/14345))
 - Migrate client transports to Apache HttpClient / Core 5.x ([#4459](https://github.com/opensearch-project/OpenSearch/pull/4459))
 - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773))
 - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java
@@ -110,7 +110,7 @@ public void execute(Task t) {
                     if (BuildParams.getRuntimeJavaVersion() == JavaVersion.VERSION_1_8) {
                         test.systemProperty("java.locale.providers", "SPI,JRE");
                     } else {
-                        test.systemProperty("java.locale.providers", "SPI,COMPAT");
+                        test.systemProperty("java.locale.providers", "SPI,CLDR");
                         if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) < 0) {
                             test.jvmArgs("--illegal-access=warn");
                         }
diff --git a/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java b/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java
@@ -105,13 +105,8 @@ private static String javaLocaleProviders() {
            SPI setting is used to allow loading custom CalendarDataProvider
            in jdk8 it has to be loaded from jre/lib/ext,
            in jdk9+ it is already within ES project and on a classpath
-
-           Due to internationalization enhancements in JDK 9 OpenSearch need to set the provider to COMPAT otherwise time/date
-           parsing will break in an incompatible way for some date patterns and locales.
-           //TODO COMPAT will be deprecated in at some point, see please https://bugs.openjdk.java.net/browse/JDK-8232906
-          See also: documentation in <code>server/org.opensearch.common.time.IsoCalendarDataProvider</code>
          */
-        return "-Djava.locale.providers=SPI,COMPAT";
+        return "-Djava.locale.providers=SPI,CLDR";
     }
 
 }
diff --git a/gradle/ide.gradle b/gradle/ide.gradle
@@ -81,7 +81,7 @@ if (System.getProperty('idea.active') == 'true') {
         }
         runConfigurations {
           defaults(JUnit) {
-            vmParameters = '-ea -Djava.locale.providers=SPI,COMPAT'
+            vmParameters = '-ea -Djava.locale.providers=SPI,CLDR'
             if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) {
               vmParameters += ' -Djava.security.manager=allow'
             }
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java
@@ -1914,14 +1914,8 @@ public void testRangeQueryWithTimeZone() throws Exception {
      * Test range with a custom locale, e.g. "de" in this case. Documents here mention the day of week
      * as "Mi" for "Mittwoch (Wednesday" and "Do" for "Donnerstag (Thursday)" and the month in the query
      * as "Dez" for "Dezember (December)".
-     * Note: this test currently needs the JVM arg `-Djava.locale.providers=SPI,COMPAT` to be set.
-     * When running with gradle this is done implicitly through the BuildPlugin, but when running from
-     * an IDE this might need to be set manually in the run configuration. See also CONTRIBUTING.md section
-     * on "Configuring IDEs And Running Tests".
      */
     public void testRangeQueryWithLocaleMapping() throws Exception {
-        assert ("SPI,COMPAT".equals(System.getProperty("java.locale.providers"))) : "`-Djava.locale.providers=SPI,COMPAT` needs to be set";
-
         assertAcked(
             prepareCreate("test").setMapping(
                 jsonBuilder().startObject()
@@ -1938,17 +1932,21 @@ public void testRangeQueryWithLocaleMapping() throws Exception {
 
         indexRandom(
             true,
-            client().prepareIndex("test").setId("1").setSource("date_field", "Mi, 06 Dez 2000 02:55:00 -0800"),
-            client().prepareIndex("test").setId("2").setSource("date_field", "Do, 07 Dez 2000 02:55:00 -0800")
+            client().prepareIndex("test").setId("1").setSource("date_field", "Mi., 06 Dez. 2000 02:55:00 -0800"),
+            client().prepareIndex("test").setId("2").setSource("date_field", "Do., 07 Dez. 2000 02:55:00 -0800")
         );
 
         SearchResponse searchResponse = client().prepareSearch("test")
-            .setQuery(QueryBuilders.rangeQuery("date_field").gte("Di, 05 Dez 2000 02:55:00 -0800").lte("Do, 07 Dez 2000 00:00:00 -0800"))
+            .setQuery(
+                QueryBuilders.rangeQuery("date_field").gte("Di., 05 Dez. 2000 02:55:00 -0800").lte("Do., 07 Dez. 2000 00:00:00 -0800")
+            )
             .get();
         assertHitCount(searchResponse, 1L);
 
         searchResponse = client().prepareSearch("test")
-            .setQuery(QueryBuilders.rangeQuery("date_field").gte("Di, 05 Dez 2000 02:55:00 -0800").lte("Fr, 08 Dez 2000 00:00:00 -0800"))
+            .setQuery(
+                QueryBuilders.rangeQuery("date_field").gte("Di., 05 Dez. 2000 02:55:00 -0800").lte("Fr., 08 Dez. 2000 00:00:00 -0800")
+            )
             .get();
         assertHitCount(searchResponse, 2L);
     }
diff --git a/server/src/test/java/org/opensearch/common/joda/JavaJodaTimeDuellingTests.java b/server/src/test/java/org/opensearch/common/joda/JavaJodaTimeDuellingTests.java
@@ -51,6 +51,7 @@
 import java.util.Locale;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 
@@ -186,7 +187,7 @@ public void testIncompatiblePatterns() {
             .parseDateTime("2019-01-01T01:01:01.001+0000");
         String jodaZoneId = DateTimeFormat.forPattern("YYYY-MM-dd'T'HH:mm:ss.SSSz").print(dateTime);
         assertThat(javaZoneId, equalTo("2019-01-01T01:01:01.001Z"));
-        assertThat(jodaZoneId, equalTo("2019-01-01T01:01:01.001UTC"));
+        assertThat(jodaZoneId, startsWith("2019-01-01T01:01:01.001"));
     }
 
     private void assertSameMillis(String input, String jodaFormat, String javaFormat) {
diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java
@@ -208,7 +208,7 @@ public void testChangeLocale() throws IOException {
             fieldMapping(b -> b.field("type", "date").field("format", "E, d MMM yyyy HH:mm:ss Z").field("locale", "de"))
         );
 
-        mapper.parse(source(b -> b.field("field", "Mi, 06 Dez 2000 02:55:00 -0800")));
+        mapper.parse(source(b -> b.field("field", "Mi., 06 Dez. 2000 02:55:00 -0800")));
     }
 
     public void testNullValue() throws IOException {

Original file line number	Diff line number	Diff line change
`@@ -110,7 +110,7 @@ public void execute(Task t) {`
`110`	`110`	`if (BuildParams.getRuntimeJavaVersion() == JavaVersion.VERSION_1_8) {`
`111`	`111`	`test.systemProperty("java.locale.providers", "SPI,JRE");`
`112`	`112`	`} else {`
`113`		`- test.systemProperty("java.locale.providers", "SPI,COMPAT");`
	`113`	`+ test.systemProperty("java.locale.providers", "SPI,CLDR");`
`114`	`114`	`if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) < 0) {`
`115`	`115`	`test.jvmArgs("--illegal-access=warn");`
`116`	`116`	`}`
Original file line number	Diff line number	Diff line change
`@@ -105,13 +105,8 @@ private static String javaLocaleProviders() {`
`105`	`105`	`SPI setting is used to allow loading custom CalendarDataProvider`
`106`	`106`	`in jdk8 it has to be loaded from jre/lib/ext,`
`107`	`107`	`in jdk9+ it is already within ES project and on a classpath`
`108`		`-`
`109`		`- Due to internationalization enhancements in JDK 9 OpenSearch need to set the provider to COMPAT otherwise time/date`
`110`		`- parsing will break in an incompatible way for some date patterns and locales.`
`111`		`- //TODO COMPAT will be deprecated in at some point, see please https://bugs.openjdk.java.net/browse/JDK-8232906`
`112`		`- See also: documentation in <code>server/org.opensearch.common.time.IsoCalendarDataProvider</code>`
`113`	`108`	`*/`
`114`		`- return "-Djava.locale.providers=SPI,COMPAT";`
	`109`	`+ return "-Djava.locale.providers=SPI,CLDR";`
`115`	`110`	`}`
`116`	`111`
`117`	`112`	`}`
Original file line number	Diff line number	Diff line change
`@@ -81,7 +81,7 @@ if (System.getProperty('idea.active') == 'true') {`
`81`	`81`	`}`
`82`	`82`	`runConfigurations {`
`83`	`83`	`defaults(JUnit) {`
`84`		`- vmParameters = '-ea -Djava.locale.providers=SPI,COMPAT'`
	`84`	`+ vmParameters = '-ea -Djava.locale.providers=SPI,CLDR'`
`85`	`85`	`if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) {`
`86`	`86`	`vmParameters += ' -Djava.security.manager=allow'`
`87`	`87`	`}`
Original file line number	Diff line number	Diff line change
`@@ -208,7 +208,7 @@ public void testChangeLocale() throws IOException {`
`208`	`208`	`fieldMapping(b -> b.field("type", "date").field("format", "E, d MMM yyyy HH:mm:ss Z").field("locale", "de"))`
`209`	`209`	`);`
`210`	`210`
`211`		`- mapper.parse(source(b -> b.field("field", "Mi, 06 Dez 2000 02:55:00 -0800")));`
	`211`	`+ mapper.parse(source(b -> b.field("field", "Mi., 06 Dez. 2000 02:55:00 -0800")));`
`212`	`212`	`}`
`213`	`213`
`214`	`214`	`public void testNullValue() throws IOException {`