Skip to content

Commit ed383d8

Browse files
opensearch-trigger-bot[bot]github-actions[bot]Sachin Kale
authored
Change RemoteSegmentStoreDirectory init at given timestamp to ignore pinned timestamp setting (opensearch-project#15457) (opensearch-project#15553)
(cherry picked from commit 30ed15d) Signed-off-by: Sachin Kale <kalsac@amazon.com> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Sachin Kale <kalsac@amazon.com>
1 parent 36a5ac5 commit ed383d8

File tree

5 files changed

+90
-67
lines changed

5 files changed

+90
-67
lines changed

‎server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -391,15 +391,24 @@ public static boolean isSwitchToStrictCompatibilityMode(ClusterUpdateSettingsReq
391391
* @param pinnedTimestampSet A set of timestamps representing pinned points in time.
392392
* @param getTimestampFunction A function that extracts the timestamp from a metadata file name.
393393
* @param prefixFunction A function that extracts a tuple of prefix information from a metadata file name.
394+
* @param ignorePinnedTimestampEnabledSetting A flag to ignore pinned timestamp enabled setting
394395
* @return A set of metadata file names that are implicitly locked based on the pinned timestamps.
395396
*/
396397
public static Set<String> getPinnedTimestampLockedFiles(
397398
List<String> metadataFiles,
398399
Set<Long> pinnedTimestampSet,
399400
Function<String, Long> getTimestampFunction,
400-
Function<String, Tuple<String, String>> prefixFunction
401+
Function<String, Tuple<String, String>> prefixFunction,
402+
boolean ignorePinnedTimestampEnabledSetting
401403
) {
402-
return getPinnedTimestampLockedFiles(metadataFiles, pinnedTimestampSet, new HashMap<>(), getTimestampFunction, prefixFunction);
404+
return getPinnedTimestampLockedFiles(
405+
metadataFiles,
406+
pinnedTimestampSet,
407+
new HashMap<>(),
408+
getTimestampFunction,
409+
prefixFunction,
410+
ignorePinnedTimestampEnabledSetting
411+
);
403412
}
404413

405414
/**
@@ -431,10 +440,28 @@ public static Set<String> getPinnedTimestampLockedFiles(
431440
Map<Long, String> metadataFilePinnedTimestampMap,
432441
Function<String, Long> getTimestampFunction,
433442
Function<String, Tuple<String, String>> prefixFunction
443+
) {
444+
return getPinnedTimestampLockedFiles(
445+
metadataFiles,
446+
pinnedTimestampSet,
447+
metadataFilePinnedTimestampMap,
448+
getTimestampFunction,
449+
prefixFunction,
450+
false
451+
);
452+
}
453+
454+
private static Set<String> getPinnedTimestampLockedFiles(
455+
List<String> metadataFiles,
456+
Set<Long> pinnedTimestampSet,
457+
Map<Long, String> metadataFilePinnedTimestampMap,
458+
Function<String, Long> getTimestampFunction,
459+
Function<String, Tuple<String, String>> prefixFunction,
460+
boolean ignorePinnedTimestampEnabledSetting
434461
) {
435462
Set<String> implicitLockedFiles = new HashSet<>();
436463

437-
if (RemoteStoreSettings.isPinnedTimestampsEnabled() == false) {
464+
if (ignorePinnedTimestampEnabledSetting == false && RemoteStoreSettings.isPinnedTimestampsEnabled() == false) {
438465
return implicitLockedFiles;
439466
}
440467

‎server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,8 @@ public RemoteSegmentMetadata initializeToSpecificTimestamp(long timestamp) throw
196196
metadataFiles,
197197
Set.of(timestamp),
198198
MetadataFilenameUtils::getTimestamp,
199-
MetadataFilenameUtils::getNodeIdByPrimaryTermAndGen
199+
MetadataFilenameUtils::getNodeIdByPrimaryTermAndGen,
200+
true
200201
);
201202
if (lockedMetadataFiles.isEmpty()) {
202203
return null;

‎server/src/main/java/org/opensearch/node/Node.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,7 @@
305305
import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath;
306306
import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY;
307307
import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED;
308+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent;
308309
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreAttributePresent;
309310
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled;
310311

@@ -812,7 +813,7 @@ protected Node(
812813
remoteClusterStateCleanupManager = null;
813814
}
814815
final RemoteStorePinnedTimestampService remoteStorePinnedTimestampService;
815-
if (isRemoteStoreAttributePresent(settings) && CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.get(settings)) {
816+
if (isRemoteDataAttributePresent(settings) && CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.get(settings)) {
816817
remoteStorePinnedTimestampService = new RemoteStorePinnedTimestampService(
817818
repositoriesServiceReference::get,
818819
settings,

‎server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1141,6 +1141,62 @@ public void testMetadataFileNameOrder() {
11411141
assertEquals(14, count);
11421142
}
11431143

1144+
public void testInitializeToSpecificTimestampNoMetadataFiles() throws IOException {
1145+
when(
1146+
remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
1147+
RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
1148+
Integer.MAX_VALUE
1149+
)
1150+
).thenReturn(new ArrayList<>());
1151+
assertNull(remoteSegmentStoreDirectory.initializeToSpecificTimestamp(1234L));
1152+
}
1153+
1154+
public void testInitializeToSpecificTimestampNoMdMatchingTimestamp() throws IOException {
1155+
String metadataPrefix = "metadata__1__2__3__4__5__";
1156+
List<String> metadataFiles = new ArrayList<>();
1157+
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(2000));
1158+
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(3000));
1159+
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(4000));
1160+
1161+
when(
1162+
remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
1163+
RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
1164+
Integer.MAX_VALUE
1165+
)
1166+
).thenReturn(metadataFiles);
1167+
assertNull(remoteSegmentStoreDirectory.initializeToSpecificTimestamp(1234L));
1168+
}
1169+
1170+
public void testInitializeToSpecificTimestampMatchingMdFile() throws IOException {
1171+
String metadataPrefix = "metadata__1__2__3__4__5__";
1172+
List<String> metadataFiles = new ArrayList<>();
1173+
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(1000));
1174+
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(2000));
1175+
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(3000));
1176+
1177+
Map<String, String> metadata = new HashMap<>();
1178+
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512::" + Version.LATEST.major);
1179+
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024::" + Version.LATEST.major);
1180+
1181+
when(
1182+
remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
1183+
RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
1184+
Integer.MAX_VALUE
1185+
)
1186+
).thenReturn(metadataFiles);
1187+
when(remoteMetadataDirectory.getBlobStream(metadataPrefix + RemoteStoreUtils.invertLong(1000))).thenReturn(
1188+
createMetadataFileBytes(metadata, indexShard.getLatestReplicationCheckpoint(), segmentInfos)
1189+
);
1190+
1191+
RemoteSegmentMetadata remoteSegmentMetadata = remoteSegmentStoreDirectory.initializeToSpecificTimestamp(1234L);
1192+
assertNotNull(remoteSegmentMetadata);
1193+
Map<String, RemoteSegmentStoreDirectory.UploadedSegmentMetadata> uploadedSegments = remoteSegmentStoreDirectory
1194+
.getSegmentsUploadedToRemoteStore();
1195+
assertEquals(2, uploadedSegments.size());
1196+
assertTrue(uploadedSegments.containsKey("_0.cfe"));
1197+
assertTrue(uploadedSegments.containsKey("_0.cfs"));
1198+
}
1199+
11441200
private static class WrapperIndexOutput extends IndexOutput {
11451201
public IndexOutput indexOutput;
11461202

‎server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryWithPinnedTimestampTests.java

Lines changed: 0 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@
88

99
package org.opensearch.index.store;
1010

11-
import org.apache.lucene.util.Version;
12-
import org.opensearch.common.UUIDs;
1311
import org.opensearch.common.blobstore.BlobMetadata;
1412
import org.opensearch.common.blobstore.BlobPath;
1513
import org.opensearch.common.blobstore.support.PlainBlobMetadata;
@@ -18,8 +16,6 @@
1816
import org.opensearch.core.action.ActionListener;
1917
import org.opensearch.gateway.remote.model.RemotePinnedTimestamps;
2018
import org.opensearch.gateway.remote.model.RemoteStorePinnedTimestampsBlobStore;
21-
import org.opensearch.index.remote.RemoteStoreUtils;
22-
import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata;
2319
import org.opensearch.index.translog.transfer.BlobStoreTransferService;
2420
import org.opensearch.indices.RemoteStoreSettings;
2521
import org.opensearch.node.Node;
@@ -31,7 +27,6 @@
3127

3228
import java.io.IOException;
3329
import java.util.ArrayList;
34-
import java.util.HashMap;
3530
import java.util.List;
3631
import java.util.Map;
3732
import java.util.function.Supplier;
@@ -40,7 +35,6 @@
4035
import org.mockito.Mockito;
4136

4237
import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED;
43-
import static org.opensearch.test.RemoteStoreTestUtils.createMetadataFileBytes;
4438
import static org.hamcrest.CoreMatchers.is;
4539
import static org.mockito.ArgumentMatchers.any;
4640
import static org.mockito.Mockito.anyInt;
@@ -143,62 +137,6 @@ private void metadataWithOlderTimestamp() {
143137
);
144138
}
145139

146-
public void testInitializeToSpecificTimestampNoMetadataFiles() throws IOException {
147-
when(
148-
remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
149-
RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
150-
Integer.MAX_VALUE
151-
)
152-
).thenReturn(new ArrayList<>());
153-
assertNull(remoteSegmentStoreDirectory.initializeToSpecificTimestamp(1234L));
154-
}
155-
156-
public void testInitializeToSpecificTimestampNoMdMatchingTimestamp() throws IOException {
157-
String metadataPrefix = "metadata__1__2__3__4__5__";
158-
List<String> metadataFiles = new ArrayList<>();
159-
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(2000));
160-
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(3000));
161-
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(4000));
162-
163-
when(
164-
remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
165-
RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
166-
Integer.MAX_VALUE
167-
)
168-
).thenReturn(metadataFiles);
169-
assertNull(remoteSegmentStoreDirectory.initializeToSpecificTimestamp(1234L));
170-
}
171-
172-
public void testInitializeToSpecificTimestampMatchingMdFile() throws IOException {
173-
String metadataPrefix = "metadata__1__2__3__4__5__";
174-
List<String> metadataFiles = new ArrayList<>();
175-
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(1000));
176-
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(2000));
177-
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(3000));
178-
179-
Map<String, String> metadata = new HashMap<>();
180-
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512::" + Version.LATEST.major);
181-
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024::" + Version.LATEST.major);
182-
183-
when(
184-
remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
185-
RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
186-
Integer.MAX_VALUE
187-
)
188-
).thenReturn(metadataFiles);
189-
when(remoteMetadataDirectory.getBlobStream(metadataPrefix + RemoteStoreUtils.invertLong(1000))).thenReturn(
190-
createMetadataFileBytes(metadata, indexShard.getLatestReplicationCheckpoint(), segmentInfos)
191-
);
192-
193-
RemoteSegmentMetadata remoteSegmentMetadata = remoteSegmentStoreDirectory.initializeToSpecificTimestamp(1234L);
194-
assertNotNull(remoteSegmentMetadata);
195-
Map<String, RemoteSegmentStoreDirectory.UploadedSegmentMetadata> uploadedSegments = remoteSegmentStoreDirectory
196-
.getSegmentsUploadedToRemoteStore();
197-
assertEquals(2, uploadedSegments.size());
198-
assertTrue(uploadedSegments.containsKey("_0.cfe"));
199-
assertTrue(uploadedSegments.containsKey("_0.cfs"));
200-
}
201-
202140
public void testDeleteStaleCommitsNoPinnedTimestampMdFilesLatest() throws Exception {
203141
metadataFilename = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
204142
12,

0 commit comments

Comments
 (0)