[Remote Store] Add support for randomizing Remote Store enabled testing. (opensearch-project#12488)

* Add support for randomizing remote store enabled testing.

Signed-off-by: Rishikesh1159 <rishireddy1159@gmail.com>

* fix spotless check

Signed-off-by: Rishikesh1159 <rishireddy1159@gmail.com>

* Updating logic to randomly use remote store once per cluster instead of once per node.

Signed-off-by: Rishikesh1159 <rishireddy1159@gmail.com>

* Add Remote Store Randomization within Replication Type Randomization.

Signed-off-by: Rishikesh1159 <rishireddy1159@gmail.com>

---------

Signed-off-by: Rishikesh1159 <rishireddy1159@gmail.com>

Author: Rishikesh1159

server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java

@@ -650,6 +650,7 @@ public void testCacheWithInvalidation() throws Exception {
                         .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", -1)
                 )
                 .get()
         );

server/src/internalClusterTest/java/org/opensearch/indices/recovery/ReplicaToPrimaryPromotionIT.java

@@ -56,6 +56,11 @@ protected int numberOfReplicas() {
         return 1;
     }
 
+    @Override
+    public boolean useRandomReplicationStrategy() {
+        return true;
+    }
+
     public void testPromoteReplicaToPrimary() throws Exception {
         final String indexName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
         createIndex(indexName);
@@ -65,7 +70,7 @@ public void testPromoteReplicaToPrimary() throws Exception {
             try (BackgroundIndexer indexer = new BackgroundIndexer(indexName, "_doc", client(), numOfDocs)) {
                 waitForDocs(numOfDocs, indexer);
             }
-            refresh(indexName);
+            refreshAndWaitForReplication(indexName);
         }
 
         assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numOfDocs);

test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java

@@ -379,6 +379,14 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase {
      */
     public static final String TESTS_CLUSTER_NAME = "tests.clustername";
 
+    protected static final String REMOTE_BACKED_STORAGE_REPOSITORY_NAME = "test-remote-store-repo";
+
+    private Path remoteStoreRepositoryPath;
+
+    private ReplicationType randomReplicationType;
+
+    private String randomStorageType;
+
     @BeforeClass
     public static void beforeClass() throws Exception {
         testClusterRule.beforeClass();
@@ -1896,11 +1904,19 @@ protected Settings nodeSettings(int nodeOrdinal) {
             builder.put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true);
         }
 
-        // Randomly set a replication strategy for the node. Replication Strategy can still be manually overridden by subclass if needed.
+        // Randomly set a Replication Strategy and storage type for the node. Both Replication Strategy and Storage Type can still be
+        // manually overridden by subclass if needed.
         if (useRandomReplicationStrategy()) {
-            ReplicationType replicationType = randomBoolean() ? ReplicationType.DOCUMENT : ReplicationType.SEGMENT;
-            logger.info("Randomly using Replication Strategy as {}.", replicationType.toString());
-            builder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), replicationType);
+            if (randomReplicationType.equals(ReplicationType.SEGMENT) && randomStorageType.equals("REMOTE_STORE")) {
+                logger.info("Randomly using Replication Strategy as {} and Storage Type as {}.", randomReplicationType, randomStorageType);
+                if (remoteStoreRepositoryPath == null) {
+                    remoteStoreRepositoryPath = randomRepoPath().toAbsolutePath();
+                }
+                builder.put(remoteStoreClusterSettings(REMOTE_BACKED_STORAGE_REPOSITORY_NAME, remoteStoreRepositoryPath));
+            } else {
+                logger.info("Randomly using Replication Strategy as {} and Storage Type as {}.", randomReplicationType, randomStorageType);
+                builder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), randomReplicationType);
+            }
         }
         return builder.build();
     }
@@ -1953,6 +1969,14 @@ protected boolean ignoreExternalCluster() {
     }
 
     protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException {
+        if (useRandomReplicationStrategy()) {
+            randomReplicationType = randomBoolean() ? ReplicationType.DOCUMENT : ReplicationType.SEGMENT;
+            if (randomReplicationType.equals(ReplicationType.SEGMENT)) {
+                randomStorageType = randomBoolean() ? "REMOTE_STORE" : "LOCAL";
+            } else {
+                randomStorageType = "LOCAL";
+            }
+        }
         String clusterAddresses = System.getProperty(TESTS_CLUSTER);
         if (Strings.hasLength(clusterAddresses) && ignoreExternalCluster() == false) {
             if (scope == Scope.TEST) {