Skip to content

Commit 6c063d3

Browse files
author
Swetha Guptha
committed
Remote publication enabled it run.
Signed-off-by: Swetha Guptha <gupthasg@amazon.com>
1 parent a98a5de commit 6c063d3

File tree

6 files changed

+68
-23
lines changed

6 files changed

+68
-23
lines changed

‎server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,8 @@
2727
import java.util.stream.Stream;
2828

2929
import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
30-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
31-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
32-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
33-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY;
30+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
31+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
3432
import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
3533
import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
3634
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -68,35 +66,38 @@ public void setUp() throws Exception {
6866
}
6967

7068
public Settings.Builder remotePublishConfiguredNodeSetting() {
69+
String remoteStoreNodeAttributePrefix = "remote_publication";
7170
String stateRepoSettingsAttributeKeyPrefix = String.format(
7271
Locale.getDefault(),
73-
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
72+
"node.attr." + REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
73+
remoteStoreNodeAttributePrefix,
7474
REPOSITORY_NAME
7575
);
7676
String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey();
7777
String stateRepoTypeAttributeKey = String.format(
7878
Locale.getDefault(),
79-
"node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
79+
"node.attr." + REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
80+
remoteStoreNodeAttributePrefix,
8081
REPOSITORY_NAME
8182
);
8283
String routingTableRepoTypeAttributeKey = String.format(
8384
Locale.getDefault(),
84-
"node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
85+
"node.attr." + REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
8586
ROUTING_TABLE_REPO_NAME
8687
);
8788
String routingTableRepoSettingsAttributeKeyPrefix = String.format(
8889
Locale.getDefault(),
89-
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
90+
"node.attr." + REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
9091
ROUTING_TABLE_REPO_NAME
9192
);
9293

9394
Settings.Builder builder = Settings.builder()
94-
.put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME)
95+
.put("node.attr." + "remote_publication.state.repository", REPOSITORY_NAME)
9596
.put(stateRepoTypeAttributeKey, ReloadableFsRepository.TYPE)
9697
.put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath)
9798
.put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, true)
9899
.put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
99-
.put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, ROUTING_TABLE_REPO_NAME)
100+
.put("node.attr." + "remote_publication.routing_table.repository", ROUTING_TABLE_REPO_NAME)
100101
.put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE)
101102
.put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath);
102103
return builder;

‎server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@
3838
import org.opensearch.test.OpenSearchIntegTestCase;
3939
import org.junit.Before;
4040

41-
import java.nio.file.Path;
4241
import java.util.HashMap;
4342
import java.util.List;
4443
import java.util.Map;
@@ -61,8 +60,6 @@ public class MigrationBaseTestCase extends OpenSearchIntegTestCase {
6160

6261
protected static final String REPOSITORY_2_NAME = "test-remote-store-repo-2";
6362

64-
protected Path segmentRepoPath;
65-
protected Path translogRepoPath;
6663
boolean addRemote = false;
6764
Settings extraSettings = Settings.EMPTY;
6865

@@ -94,7 +91,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
9491
return Settings.builder()
9592
.put(super.nodeSettings(nodeOrdinal))
9693
.put(extraSettings)
97-
.put(remoteStoreClusterSettings(REPOSITORY_NAME, segmentRepoPath, REPOSITORY_2_NAME, translogRepoPath))
94+
.put(remoteStoreClusterSettings(REPOSITORY_NAME, super.segmentRepoPath, REPOSITORY_2_NAME, super.translogRepoPath))
9895
.put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
9996
.build();
10097
} else {

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -493,7 +493,9 @@ private Path registerCustomRepository() {
493493

494494
private void verifyRestoredRepositories(Path repoPath) {
495495
RepositoriesMetadata repositoriesMetadata = clusterService().state().metadata().custom(RepositoriesMetadata.TYPE);
496-
assertEquals(3, repositoriesMetadata.repositories().size()); // includes remote store repo as well
496+
assertEquals(3, repositoriesMetadata.repositories().size());
497+
// routing repo added
498+
assertEquals(4, repositoriesMetadata.repositories().size()); // includes remote store repo as well
497499
assertTrue(SYSTEM_REPOSITORY_SETTING.get(repositoriesMetadata.repository(REPOSITORY_NAME).settings()));
498500
assertTrue(SYSTEM_REPOSITORY_SETTING.get(repositoriesMetadata.repository(REPOSITORY_2_NAME).settings()));
499501
assertEquals("fs", repositoriesMetadata.repository("custom-repo").type());

‎server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -552,7 +552,11 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod
552552
List<String> reposToSkip = new ArrayList<>(1);
553553
// find a remote node which has routing table configured
554554
Optional<DiscoveryNode> remoteRoutingTableNode = existingNodes.stream()
555-
.filter(node -> node.isRemoteStoreNode() && RemoteStoreNodeAttribute.getRoutingTableRepoName(node.getAttributes()) != null)
555+
.filter(
556+
node -> node.isRemoteStoreNode()
557+
&& RemoteStoreNodeAttribute.getRoutingTableRepoName(node.getAttributes()) != null
558+
&& node.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY) != null
559+
)
556560
.findFirst();
557561
// If none of the existing nodes have routing table repo, then we skip this repo check if present in joining node.
558562
// This ensures a new node with remote routing table repo is able to join the cluster.

‎test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,11 @@ public void assertRepoConsistency() {
161161
.get()
162162
.repositories()
163163
.stream()
164-
.filter(repositoryMetadata -> !repositoryMetadata.name().endsWith(TEST_REMOTE_STORE_REPO_SUFFIX))
164+
.filter(
165+
repositoryMetadata -> !repositoryMetadata.name().endsWith(TEST_REMOTE_STORE_REPO_SUFFIX)
166+
&& !repositoryMetadata.name().equals("test-remote-store-repo")
167+
&& !repositoryMetadata.name().equals("remote-routing-repo")
168+
)
165169
.forEach(repositoryMetadata -> {
166170
final String name = repositoryMetadata.name();
167171
if (repositoryMetadata.settings().getAsBoolean("readonly", false) == false) {

‎test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java

Lines changed: 43 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -220,18 +220,15 @@
220220
import static org.opensearch.core.common.util.CollectionUtils.eagerPartition;
221221
import static org.opensearch.discovery.DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING;
222222
import static org.opensearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING;
223+
import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
224+
import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY;
223225
import static org.opensearch.index.IndexSettings.INDEX_DOC_ID_FUZZY_SET_ENABLED_SETTING;
224226
import static org.opensearch.index.IndexSettings.INDEX_DOC_ID_FUZZY_SET_FALSE_POSITIVE_PROBABILITY_SETTING;
225227
import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING;
226228
import static org.opensearch.index.query.QueryBuilders.matchAllQuery;
227229
import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1;
228230
import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING;
229-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
230-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
231-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
232-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY;
233-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
234-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
231+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.*;
235232
import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
236233
import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
237234
import static org.opensearch.test.XContentTestUtils.convertToMap;
@@ -411,6 +408,9 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase {
411408

412409
private String randomStorageType;
413410

411+
protected Path translogRepoPath;
412+
protected Path segmentRepoPath;
413+
414414
@BeforeClass
415415
public static void beforeClass() throws Exception {
416416
prefixModeVerificationEnable = randomBoolean();
@@ -1968,6 +1968,43 @@ protected Settings nodeSettings(int nodeOrdinal) {
19681968
builder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), randomReplicationType);
19691969
}
19701970
}
1971+
1972+
if (segmentRepoPath == null || translogRepoPath == null) {
1973+
segmentRepoPath = randomRepoPath().toAbsolutePath();
1974+
translogRepoPath = randomRepoPath().toAbsolutePath();
1975+
}
1976+
String segmentRepoName = "test-remote-store-repo";
1977+
String stateRepoSettingsAttributeKeyPrefix = String.format(
1978+
Locale.getDefault(),
1979+
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
1980+
segmentRepoName
1981+
);
1982+
String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey();
1983+
String stateRepoTypeAttributeKey = String.format(
1984+
Locale.getDefault(),
1985+
"node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
1986+
segmentRepoName
1987+
);
1988+
String routingTableRepoName = "remote-routing-repo";
1989+
String routingTableRepoTypeAttributeKey = String.format(
1990+
Locale.getDefault(),
1991+
"node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
1992+
routingTableRepoName
1993+
);
1994+
String routingTableRepoSettingsAttributeKeyPrefix = String.format(
1995+
Locale.getDefault(),
1996+
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
1997+
routingTableRepoName
1998+
);
1999+
builder.put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName)
2000+
.put(stateRepoTypeAttributeKey, ReloadableFsRepository.TYPE)
2001+
.put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath)
2002+
.put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, prefixModeVerificationEnable)
2003+
.put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
2004+
.put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, routingTableRepoName)
2005+
.put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE)
2006+
.put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath)
2007+
.put(REMOTE_PUBLICATION_SETTING_KEY, true);
19712008
return builder.build();
19722009
}
19732010

0 commit comments

Comments
 (0)