Flaky test : Don't use async repo for SplitIndex and wait for translog file deletion

Signed-off-by: Gaurav Bafna <gbbafna@amazon.com>

Author: gbbafna

server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteSplitIndexIT.java

@@ -69,6 +69,7 @@
 import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.VersionUtils;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -89,6 +90,10 @@
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteSplitIndexIT extends RemoteStoreBaseIntegTestCase {
+    @Before
+    public void setup() {
+        asyncUploadMockFsRepo = false;
+    }
 
     @Override
     protected boolean forbidPrivateIndexSettings() {

server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java

@@ -19,6 +19,7 @@
 import org.opensearch.action.index.IndexRequest;
 import org.opensearch.action.index.IndexRequestBuilder;
 import org.opensearch.action.index.IndexResponse;
+import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.action.support.WriteRequest;
 import org.opensearch.cluster.metadata.IndexMetadata;
@@ -57,6 +58,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -257,14 +259,25 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas, long totalFiel
     }
 
     @After
-    public void teardown() {
+    public void teardown() throws Exception {
         clusterSettingsSuppliedByTest = false;
         asyncUploadMockFsRepo = randomBoolean();
         metadataSupportedType = randomBoolean();
         assertRemoteStoreRepositoryOnAllNodes(REPOSITORY_NAME);
         assertRemoteStoreRepositoryOnAllNodes(REPOSITORY_2_NAME);
         clusterAdmin().prepareCleanupRepository(REPOSITORY_NAME).get();
         clusterAdmin().prepareCleanupRepository(REPOSITORY_2_NAME).get();
+        // Delete is async.
+        assertAcked(
+            client().admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get()
+        );
+        assertBusy(() -> {
+            try {
+                assertEquals(0, getFileCount(translogRepoPath));
+            } catch (IOException e) {
+                fail();
+            }
+        }, 30, TimeUnit.SECONDS);
     }
 
     public RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) {
@@ -313,7 +326,7 @@ public void assertRemoteStoreRepositoryOnAllNodes(String repositoryName) {
         }
     }
 
-    public static int getFileCount(Path path) throws Exception {
+    public static int getFileCount(Path path) throws IOException {
         final AtomicInteger filesExisting = new AtomicInteger(0);
         Files.walkFileTree(path, new SimpleFileVisitor<>() {
             @Override