Fix flaky tests in GoogleCloudStorageBlobStoreRepositoryTests, S3BlobStoreRepositoryTests, AzureBlobStoreRepositoryTests (#18290)

Signed-off-by: kkewwei <kewei.11@bytedance.com>
Signed-off-by: kkewwei <kkewwei@163.com>
(cherry picked from commit bde7db5)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: github-actions[bot]

plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java

@@ -95,7 +95,7 @@ protected Map<String, HttpHandler> createHttpHandlers() {
 
     @Override
     protected HttpHandler createErroneousHttpHandler(final HttpHandler delegate) {
-        return new AzureErroneousHttpHandler(delegate, randomIntBetween(2, 3));
+        return new AzureErroneousHttpHandler(delegate, randomDoubleBetween(0, 0.25, false));
     }
 
     @Override
@@ -163,8 +163,8 @@ private static class AzureBlobStoreHttpHandler extends AzureHttpHandler implemen
     @SuppressForbidden(reason = "this test uses a HttpServer to emulate an Azure endpoint")
     private static class AzureErroneousHttpHandler extends ErroneousHttpHandler {
 
-        AzureErroneousHttpHandler(final HttpHandler delegate, final int maxErrorsPerRequest) {
-            super(delegate, maxErrorsPerRequest);
+        AzureErroneousHttpHandler(final HttpHandler delegate, final double maxErrorsPercentage) {
+            super(delegate, maxErrorsPercentage);
         }
 
         @Override

plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java

@@ -112,7 +112,7 @@ protected Map<String, HttpHandler> createHttpHandlers() {
 
     @Override
     protected HttpHandler createErroneousHttpHandler(final HttpHandler delegate) {
-        return new GoogleErroneousHttpHandler(delegate, randomIntBetween(2, 3));
+        return new GoogleErroneousHttpHandler(delegate, randomDoubleBetween(0, 0.25, false));
     }
 
     @Override
@@ -305,8 +305,8 @@ private static class GoogleCloudStorageBlobStoreHttpHandler extends GoogleCloudS
     @SuppressForbidden(reason = "this test uses a HttpServer to emulate a Google Cloud Storage endpoint")
     private static class GoogleErroneousHttpHandler extends ErroneousHttpHandler {
 
-        GoogleErroneousHttpHandler(final HttpHandler delegate, final int maxErrorsPerRequest) {
-            super(delegate, maxErrorsPerRequest);
+        GoogleErroneousHttpHandler(final HttpHandler delegate, final double maxErrorsPercentage) {
+            super(delegate, maxErrorsPercentage);
         }
 
         @Override

plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -150,7 +150,7 @@ protected Map<String, HttpHandler> createHttpHandlers() {
 
     @Override
     protected HttpHandler createErroneousHttpHandler(final HttpHandler delegate) {
-        return new S3StatsCollectorHttpHandler(new S3ErroneousHttpHandler(delegate, randomIntBetween(2, 3)));
+        return new S3StatsCollectorHttpHandler(new S3ErroneousHttpHandler(delegate, randomDoubleBetween(0, 0.25, false)));
     }
 
     @Override
@@ -395,8 +395,8 @@ private void validateAuthHeader(HttpExchange exchange) {
     @SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint")
     private static class S3ErroneousHttpHandler extends ErroneousHttpHandler {
 
-        S3ErroneousHttpHandler(final HttpHandler delegate, final int maxErrorsPerRequest) {
-            super(delegate, maxErrorsPerRequest);
+        S3ErroneousHttpHandler(final HttpHandler delegate, final double maxErrorsPercentage) {
+            super(delegate, maxErrorsPercentage);
         }
 
         @Override

test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchMockAPIBasedRepositoryIntegTestCase.java

@@ -277,14 +277,15 @@ protected abstract static class ErroneousHttpHandler implements DelegatingHttpHa
         private final Map<String, AtomicInteger> requests;
 
         private final HttpHandler delegate;
-        private final int maxErrorsPerRequest;
+        private final double maxErrorsPercentage;
 
         @SuppressForbidden(reason = "this test uses a HttpServer to emulate a cloud-based storage service")
-        protected ErroneousHttpHandler(final HttpHandler delegate, final int maxErrorsPerRequest) {
+        protected ErroneousHttpHandler(final HttpHandler delegate, final double maxErrorsPercentage) {
             this.requests = new ConcurrentHashMap<>();
             this.delegate = delegate;
-            this.maxErrorsPerRequest = maxErrorsPerRequest;
-            assert maxErrorsPerRequest > 1;
+            this.maxErrorsPercentage = maxErrorsPercentage;
+            // We don't want to fail too often as it will cost too much time, which will lead to flaky tests
+            assert maxErrorsPercentage >= 0 && maxErrorsPercentage <= 0.25;
         }
 
         @Override
@@ -295,7 +296,9 @@ public void handle(final HttpExchange exchange) throws IOException {
 
                 final boolean canFailRequest = canFailRequest(exchange);
                 final int count = requests.computeIfAbsent(requestId, req -> new AtomicInteger(0)).incrementAndGet();
-                if (count >= maxErrorsPerRequest || canFailRequest == false) {
+                // We should not fail more than 3 times as the default max retry count is 3 (see SdkDefaultRetrySetting.maxAttempts), the
+                // request will fail when retry count > 3.
+                if (random().nextDouble() > maxErrorsPercentage || count >= 3 || canFailRequest == false) {
                     requests.remove(requestId);
                     delegate.handle(exchange);
                 } else {