Skip to content

Commit ddc8fb2

Browse files
ltaragiltaragi
committed
Add IT
Signed-off-by: Lakshya Taragi <lakshya.taragi@gmail.com>
1 parent d915bcd commit ddc8fb2

File tree

3 files changed

+289
-31
lines changed

3 files changed

+289
-31
lines changed

‎server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,6 @@
1414
import org.opensearch.common.settings.Settings;
1515
import org.opensearch.common.settings.SettingsException;
1616
import org.opensearch.core.rest.RestStatus;
17-
import org.opensearch.index.IndexSettings;
18-
import org.opensearch.indices.replication.common.ReplicationType;
1917
import org.opensearch.snapshots.SnapshotInfo;
2018
import org.opensearch.snapshots.SnapshotState;
2119
import org.opensearch.test.InternalTestCluster;
@@ -24,11 +22,6 @@
2422
import java.nio.file.Path;
2523
import java.util.Optional;
2624

27-
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
28-
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
29-
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
30-
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
31-
import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
3225
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.MIXED;
3326
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT;
3427
import static org.opensearch.node.remotestore.RemoteStoreNodeService.Direction.REMOTE_STORE;
@@ -189,28 +182,6 @@ private void restoreSnapshot(String snapshotRepoName, String snapshotName, Strin
189182
ensureGreen(restoredIndexName);
190183
}
191184

192-
// verify that the created index is not remote store backed
193-
private void assertNonRemoteStoreBackedIndex(String indexName) {
194-
Settings indexSettings = client.admin().indices().prepareGetIndex().execute().actionGet().getSettings().get(indexName);
195-
assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
196-
assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
197-
assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
198-
assertNull(indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
199-
}
200-
201-
// verify that the created index is remote store backed
202-
private void assertRemoteStoreBackedIndex(String indexName) {
203-
Settings indexSettings = client.admin().indices().prepareGetIndex().execute().actionGet().getSettings().get(indexName);
204-
assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
205-
assertEquals("true", indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
206-
assertEquals(REPOSITORY_NAME, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
207-
assertEquals(REPOSITORY_2_NAME, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
208-
assertEquals(
209-
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL,
210-
INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)
211-
);
212-
}
213-
214185
// bootstrap a cluster
215186
private void initializeCluster(boolean remoteClusterManager) {
216187
addRemote = remoteClusterManager;

‎server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java

Lines changed: 183 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,32 @@
88

99
package org.opensearch.remotemigration;
1010

11+
import org.opensearch.action.admin.cluster.allocation.ClusterAllocationExplanation;
1112
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
13+
import org.opensearch.action.support.ActiveShardCount;
1214
import org.opensearch.cluster.node.DiscoveryNode;
1315
import org.opensearch.cluster.node.DiscoveryNodes;
1416
import org.opensearch.cluster.routing.IndexShardRoutingTable;
1517
import org.opensearch.cluster.routing.ShardRouting;
18+
import org.opensearch.cluster.routing.ShardRoutingState;
19+
import org.opensearch.cluster.routing.allocation.AllocateUnassignedDecision;
20+
import org.opensearch.cluster.routing.allocation.MoveDecision;
21+
import org.opensearch.cluster.routing.allocation.NodeAllocationResult;
22+
import org.opensearch.cluster.routing.allocation.decider.Decision;
23+
import org.opensearch.common.Nullable;
1624
import org.opensearch.common.settings.Settings;
25+
import org.opensearch.index.IndexSettings;
26+
import org.opensearch.indices.replication.common.ReplicationType;
1727

28+
import java.util.List;
1829
import java.util.Map;
1930
import java.util.Optional;
2031

32+
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
33+
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
34+
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
35+
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
36+
import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
2137
import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
2238
import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
2339
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -35,7 +51,7 @@ protected void setClusterMode(String mode) {
3551
}
3652

3753
// set the migration direction for cluster [remote_store, docrep, none]
38-
public void setDirection(String direction) {
54+
protected void setDirection(String direction) {
3955
updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), direction));
4056
assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
4157
}
@@ -67,7 +83,7 @@ protected String allNodesExcept(String except) {
6783
return exclude.toString();
6884
}
6985

70-
// create a new test index
86+
// create a new test index with un-allocated primary and no replicas
7187
protected void prepareIndexWithoutReplica(Optional<String> name) {
7288
String indexName = name.orElse(TEST_INDEX);
7389
internalCluster().client()
@@ -84,6 +100,33 @@ protected void prepareIndexWithoutReplica(Optional<String> name) {
84100
.actionGet();
85101
}
86102

103+
// create a new test index with allocated primary and 1 unallocated replica
104+
public void prepareIndexWithAllocatedPrimary(DiscoveryNode primaryShardNode, Optional<String> name) {
105+
String indexName = name.orElse(TEST_INDEX);
106+
internalCluster().client()
107+
.admin()
108+
.indices()
109+
.prepareCreate(indexName)
110+
.setSettings(
111+
Settings.builder()
112+
.put("index.number_of_shards", 1)
113+
.put("index.number_of_replicas", 1)
114+
.put("index.routing.allocation.include._name", primaryShardNode.getName())
115+
.put("index.routing.allocation.exclude._name", allNodesExcept(primaryShardNode.getName()))
116+
)
117+
.setWaitForActiveShards(ActiveShardCount.ONE)
118+
.execute()
119+
.actionGet();
120+
121+
ensureYellowAndNoInitializingShards(TEST_INDEX);
122+
123+
logger.info(" --> verify allocation of primary shard");
124+
assertAllocation(true, primaryShardNode);
125+
126+
logger.info(" --> verify non-allocation of replica shard");
127+
assertNonAllocation(false);
128+
}
129+
87130
protected ShardRouting getShardRouting(boolean isPrimary) {
88131
IndexShardRoutingTable table = internalCluster().client()
89132
.admin()
@@ -98,4 +141,142 @@ protected ShardRouting getShardRouting(boolean isPrimary) {
98141
return (isPrimary ? table.primaryShard() : table.replicaShards().get(0));
99142
}
100143

144+
// obtain decision for allocation/relocation of a shard to a given node
145+
protected Decision getDecisionForTargetNode(
146+
DiscoveryNode targetNode,
147+
boolean isPrimary,
148+
boolean includeYesDecisions,
149+
boolean isRelocation
150+
) {
151+
ClusterAllocationExplanation explanation = internalCluster().client()
152+
.admin()
153+
.cluster()
154+
.prepareAllocationExplain()
155+
.setIndex(TEST_INDEX)
156+
.setShard(0)
157+
.setPrimary(isPrimary)
158+
.setIncludeYesDecisions(includeYesDecisions)
159+
.get()
160+
.getExplanation();
161+
162+
Decision requiredDecision = null;
163+
List<NodeAllocationResult> nodeAllocationResults;
164+
if (isRelocation) {
165+
MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
166+
nodeAllocationResults = moveDecision.getNodeDecisions();
167+
} else {
168+
AllocateUnassignedDecision allocateUnassignedDecision = explanation.getShardAllocationDecision().getAllocateDecision();
169+
nodeAllocationResults = allocateUnassignedDecision.getNodeDecisions();
170+
}
171+
172+
for (NodeAllocationResult nodeAllocationResult : nodeAllocationResults) {
173+
if (nodeAllocationResult.getNode().equals(targetNode)) {
174+
for (Decision decision : nodeAllocationResult.getCanAllocateDecision().getDecisions()) {
175+
if (decision.label().equals(NAME)) {
176+
requiredDecision = decision;
177+
break;
178+
}
179+
}
180+
}
181+
}
182+
183+
assertNotNull(requiredDecision);
184+
return requiredDecision;
185+
}
186+
187+
// get allocation and relocation decisions for all nodes
188+
protected void prepareDecisions() {
189+
internalCluster().client()
190+
.admin()
191+
.indices()
192+
.prepareUpdateSettings(TEST_INDEX)
193+
.setSettings(Settings.builder().put("index.routing.allocation.exclude._name", allNodesExcept(null)))
194+
.execute()
195+
.actionGet();
196+
}
197+
198+
protected void attemptAllocation(@Nullable String targetNodeName) {
199+
Settings.Builder settingsBuilder;
200+
if (targetNodeName != null) {
201+
settingsBuilder = Settings.builder()
202+
.put("index.routing.allocation.include._name", targetNodeName)
203+
.put("index.routing.allocation.exclude._name", allNodesExcept(targetNodeName));
204+
} else {
205+
String clusterManagerNodeName = internalCluster().client()
206+
.admin()
207+
.cluster()
208+
.prepareState()
209+
.execute()
210+
.actionGet()
211+
.getState()
212+
.getNodes()
213+
.getClusterManagerNode()
214+
.getName();
215+
// to allocate freely among all nodes other than cluster-manager node
216+
settingsBuilder = Settings.builder()
217+
.put("index.routing.allocation.include._name", allNodesExcept(clusterManagerNodeName))
218+
.put("index.routing.allocation.exclude._name", clusterManagerNodeName);
219+
}
220+
internalCluster().client().admin().indices().prepareUpdateSettings(TEST_INDEX).setSettings(settingsBuilder).execute().actionGet();
221+
}
222+
223+
// verify that shard does not exist at targetNode
224+
protected void assertNonAllocation(boolean isPrimary) {
225+
if (isPrimary) {
226+
ensureRed(TEST_INDEX);
227+
} else {
228+
ensureYellowAndNoInitializingShards(TEST_INDEX);
229+
}
230+
ShardRouting shardRouting = getShardRouting(isPrimary);
231+
assertFalse(shardRouting.active());
232+
assertNull(shardRouting.currentNodeId());
233+
assertEquals(ShardRoutingState.UNASSIGNED, shardRouting.state());
234+
}
235+
236+
// verify that shard exists at targetNode
237+
protected void assertAllocation(boolean isPrimary, @Nullable DiscoveryNode targetNode) {
238+
ShardRouting shardRouting = getShardRouting(isPrimary);
239+
assertTrue(shardRouting.active());
240+
assertNotNull(shardRouting.currentNodeId());
241+
if (targetNode != null) {
242+
assertEquals(shardRouting.currentNodeId(), targetNode.getId());
243+
}
244+
}
245+
246+
// verify that the created index is not remote store backed
247+
protected void assertNonRemoteStoreBackedIndex(String indexName) {
248+
Settings indexSettings = internalCluster().client()
249+
.admin()
250+
.indices()
251+
.prepareGetIndex()
252+
.execute()
253+
.actionGet()
254+
.getSettings()
255+
.get(indexName);
256+
assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
257+
assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
258+
assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
259+
assertNull(indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
260+
}
261+
262+
// verify that the created index is remote store backed
263+
protected void assertRemoteStoreBackedIndex(String indexName) {
264+
Settings indexSettings = internalCluster().client()
265+
.admin()
266+
.indices()
267+
.prepareGetIndex()
268+
.execute()
269+
.actionGet()
270+
.getSettings()
271+
.get(indexName);
272+
assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
273+
assertEquals("true", indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
274+
assertEquals(REPOSITORY_NAME, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
275+
assertEquals(REPOSITORY_2_NAME, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
276+
assertEquals(
277+
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL,
278+
INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)
279+
);
280+
}
281+
101282
}

‎server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationIT.java

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.remotemigration;
10+
11+
import org.opensearch.client.Client;
12+
import org.opensearch.cluster.node.DiscoveryNode;
13+
import org.opensearch.cluster.routing.ShardRouting;
14+
import org.opensearch.cluster.routing.allocation.decider.Decision;
15+
import org.opensearch.test.OpenSearchIntegTestCase;
16+
17+
import java.util.Locale;
18+
import java.util.Optional;
19+
20+
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.MIXED;
21+
22+
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
23+
public class RemoteStoreMigrationShardAllocationIT extends RemoteStoreMigrationShardAllocationBaseTestCase {
24+
25+
public static final String NAME = "remote_store_migration";
26+
27+
private Client client;
28+
29+
// test for shard allocation decisions for MIXED mode and NONE direction
30+
public void testAllocationForRemoteStoreBackedIndexForNoneDirectionAndMixedMode() throws Exception {
31+
logger.info("Initialize cluster");
32+
initializeCluster(true);
33+
34+
logger.info("Add data nodes");
35+
String remoteNodeName1 = internalCluster().startDataOnlyNode();
36+
String remoteNodeName2 = internalCluster().startDataOnlyNode();
37+
internalCluster().validateClusterFormed();
38+
DiscoveryNode remoteNode1 = assertNodeInCluster(remoteNodeName1);
39+
DiscoveryNode remoteNode2 = assertNodeInCluster(remoteNodeName2);
40+
41+
logger.info("Prepare test index");
42+
boolean isReplicaAllocation = randomBoolean();
43+
if (isReplicaAllocation) {
44+
prepareIndexWithAllocatedPrimary(remoteNode1, Optional.empty());
45+
} else {
46+
prepareIndexWithoutReplica(Optional.empty());
47+
}
48+
assertRemoteStoreBackedIndex(TEST_INDEX);
49+
50+
logger.info("Switch to MIXED cluster compatibility mode");
51+
setClusterMode(MIXED.mode);
52+
addRemote = false;
53+
String docrepNodeName = internalCluster().startDataOnlyNode();
54+
internalCluster().validateClusterFormed();
55+
DiscoveryNode docrepNode = assertNodeInCluster(docrepNodeName);
56+
57+
logger.info("Verify decision for allocation on docrep node");
58+
prepareDecisions();
59+
Decision decision = getDecisionForTargetNode(docrepNode, !isReplicaAllocation, false, false);
60+
assertEquals(Decision.Type.NO, decision.type());
61+
String expectedReason = String.format(
62+
Locale.ROOT,
63+
"[none migration_direction]: %s shard copy can not be allocated to a non-remote node for remote store backed index",
64+
(isReplicaAllocation ? "replica" : "primary")
65+
);
66+
assertEquals(expectedReason, decision.getExplanation().toLowerCase(Locale.ROOT));
67+
68+
logger.info("Attempt allocation of shard on non-remote node");
69+
attemptAllocation(docrepNodeName);
70+
71+
logger.info("Verify non-allocation of shard");
72+
assertNonAllocation(!isReplicaAllocation);
73+
74+
logger.info("Verify decision for allocation on remote node");
75+
decision = getDecisionForTargetNode(remoteNode2, !isReplicaAllocation, true, false);
76+
assertEquals(Decision.Type.YES, decision.type());
77+
expectedReason = String.format(
78+
Locale.ROOT,
79+
"[none migration_direction]: %s shard copy can be allocated to a remote node for remote store backed index",
80+
(isReplicaAllocation ? "replica" : "primary")
81+
);
82+
assertEquals(expectedReason, decision.getExplanation().toLowerCase(Locale.ROOT));
83+
84+
logger.info("Attempt free allocation of shard on remote node");
85+
attemptAllocation(null);
86+
87+
logger.info("Verify successful allocation of shard");
88+
if (!isReplicaAllocation) {
89+
ensureGreen(TEST_INDEX);
90+
} else {
91+
ensureYellowAndNoInitializingShards(TEST_INDEX);
92+
}
93+
assertAllocation(!isReplicaAllocation, null);
94+
logger.info("Verify allocation on one of the remote nodes");
95+
ShardRouting shardRouting = getShardRouting(!isReplicaAllocation);
96+
assertTrue(shardRouting.currentNodeId().equals(remoteNode1.getId()) || shardRouting.currentNodeId().equals(remoteNode2.getId()));
97+
}
98+
99+
// bootstrap a cluster
100+
private void initializeCluster(boolean remoteClusterManager) {
101+
addRemote = remoteClusterManager;
102+
internalCluster().startClusterManagerOnlyNode();
103+
client = internalCluster().client();
104+
}
105+
106+
}

0 commit comments

Comments
 (0)