Skip to content

Commit 2d8c68c

Browse files
soosinhasoosinha
authored
Add versioning for UploadedIndexMetadata (#14677)
* Add versioning for UploadedIndexMetadata * Handle componentPrefix for backward compatibility Signed-off-by: Sooraj Sinha <soosinha@amazon.com>
1 parent 125b773 commit 2d8c68c

File tree

3 files changed

+81
-21
lines changed

3 files changed

+81
-21
lines changed

‎server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java

Lines changed: 52 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.opensearch.core.xcontent.ToXContentFragment;
2121
import org.opensearch.core.xcontent.XContentBuilder;
2222
import org.opensearch.core.xcontent.XContentParser;
23+
import org.opensearch.gateway.remote.ClusterMetadataManifest.Builder;
2324

2425
import java.io.IOException;
2526
import java.util.ArrayList;
@@ -243,7 +244,7 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
243244
parser.declareBoolean(ConstructingObjectParser.constructorArg(), COMMITTED_FIELD);
244245
parser.declareObjectArray(
245246
ConstructingObjectParser.constructorArg(),
246-
(p, c) -> UploadedIndexMetadata.fromXContent(p),
247+
(p, c) -> UploadedIndexMetadata.fromXContent(p, codec_version),
247248
INDICES_FIELD
248249
);
249250
parser.declareString(ConstructingObjectParser.constructorArg(), PREVIOUS_CLUSTER_UUID);
@@ -277,7 +278,7 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
277278
parser.declareLong(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_VERSION_FIELD);
278279
parser.declareObjectArray(
279280
ConstructingObjectParser.constructorArg(),
280-
(p, c) -> UploadedIndexMetadata.fromXContent(p),
281+
(p, c) -> UploadedIndexMetadata.fromXContent(p, codec_version),
281282
INDICES_ROUTING_FIELD
282283
);
283284
parser.declareNamedObject(
@@ -1112,16 +1113,30 @@ private static String componentPrefix(Object[] fields) {
11121113
return (String) fields[3];
11131114
}
11141115

1115-
private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER = new ConstructingObjectParser<>(
1116+
private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER_V0 = new ConstructingObjectParser<>(
1117+
"uploaded_index_metadata",
1118+
fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields))
1119+
);
1120+
1121+
private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER_V2 = new ConstructingObjectParser<>(
11161122
"uploaded_index_metadata",
11171123
fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields), componentPrefix(fields))
11181124
);
11191125

1126+
private static final ConstructingObjectParser<UploadedIndexMetadata, Void> CURRENT_PARSER = PARSER_V2;
1127+
11201128
static {
1121-
PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_NAME_FIELD);
1122-
PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_UUID_FIELD);
1123-
PARSER.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD);
1124-
PARSER.declareString(ConstructingObjectParser.constructorArg(), COMPONENT_PREFIX_FIELD);
1129+
declareParser(PARSER_V0, CODEC_V0);
1130+
declareParser(PARSER_V2, CODEC_V2);
1131+
}
1132+
1133+
private static void declareParser(ConstructingObjectParser<UploadedIndexMetadata, Void> parser, long codec_version) {
1134+
parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_NAME_FIELD);
1135+
parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_UUID_FIELD);
1136+
parser.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD);
1137+
if (codec_version >= CODEC_V2) {
1138+
parser.declareString(ConstructingObjectParser.constructorArg(), COMPONENT_PREFIX_FIELD);
1139+
}
11251140
}
11261141

11271142
static final String COMPONENT_PREFIX = "index--";
@@ -1130,15 +1145,32 @@ private static String componentPrefix(Object[] fields) {
11301145
private final String indexUUID;
11311146
private final String uploadedFilename;
11321147

1148+
private long codecVersion = CODEC_V2;
1149+
11331150
public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName) {
1134-
this(indexName, indexUUID, uploadedFileName, COMPONENT_PREFIX);
1151+
this(indexName, indexUUID, uploadedFileName, CODEC_V2);
1152+
}
1153+
1154+
public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, long codecVersion) {
1155+
this(indexName, indexUUID, uploadedFileName, COMPONENT_PREFIX, codecVersion);
11351156
}
11361157

11371158
public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, String componentPrefix) {
1159+
this(indexName, indexUUID, uploadedFileName, componentPrefix, CODEC_V2);
1160+
}
1161+
1162+
public UploadedIndexMetadata(
1163+
String indexName,
1164+
String indexUUID,
1165+
String uploadedFileName,
1166+
String componentPrefix,
1167+
long codecVersion
1168+
) {
11381169
this.componentPrefix = componentPrefix;
11391170
this.indexName = indexName;
11401171
this.indexUUID = indexUUID;
11411172
this.uploadedFilename = uploadedFileName;
1173+
this.codecVersion = codecVersion;
11421174
}
11431175

11441176
public UploadedIndexMetadata(StreamInput in) throws IOException {
@@ -1175,10 +1207,13 @@ public String getComponentPrefix() {
11751207

11761208
@Override
11771209
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
1178-
return builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName())
1210+
builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName())
11791211
.field(INDEX_UUID_FIELD.getPreferredName(), getIndexUUID())
1180-
.field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath())
1181-
.field(COMPONENT_PREFIX_FIELD.getPreferredName(), getComponentPrefix());
1212+
.field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath());
1213+
if (codecVersion >= CODEC_V2) {
1214+
builder.field(COMPONENT_PREFIX_FIELD.getPreferredName(), getComponentPrefix());
1215+
}
1216+
return builder;
11821217
}
11831218

11841219
@Override
@@ -1214,9 +1249,13 @@ public String toString() {
12141249
return Strings.toString(MediaTypeRegistry.JSON, this);
12151250
}
12161251

1217-
public static UploadedIndexMetadata fromXContent(XContentParser parser) throws IOException {
1218-
return PARSER.parse(parser, null);
1252+
public static UploadedIndexMetadata fromXContent(XContentParser parser, long codecVersion) throws IOException {
1253+
if (codecVersion >= CODEC_V2) {
1254+
return CURRENT_PARSER.parse(parser, null);
1255+
}
1256+
return PARSER_V0.parse(parser, null);
12191257
}
1258+
12201259
}
12211260

12221261
/**

‎server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
public class ClusterMetadataManifestTests extends OpenSearchTestCase {
4949

5050
public void testClusterMetadataManifestXContentV0() throws IOException {
51-
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
51+
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path", CODEC_V0);
5252
ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
5353
.clusterTerm(1L)
5454
.stateVersion(1L)
@@ -74,7 +74,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException {
7474
}
7575

7676
public void testClusterMetadataManifestXContentV1() throws IOException {
77-
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
77+
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path", CODEC_V1);
7878
ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
7979
.clusterTerm(1L)
8080
.stateVersion(1L)
@@ -619,6 +619,24 @@ public void testUploadedIndexMetadataSerializationEqualsHashCode() {
619619
);
620620
}
621621

622+
public void testUploadedIndexMetadataWithoutComponentPrefix() throws IOException {
623+
final UploadedIndexMetadata originalUploadedIndexMetadata = new UploadedIndexMetadata(
624+
"test-index",
625+
"test-index-uuid",
626+
"test_file_name",
627+
CODEC_V1
628+
);
629+
final XContentBuilder builder = JsonXContent.contentBuilder();
630+
builder.startObject();
631+
originalUploadedIndexMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
632+
builder.endObject();
633+
634+
try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
635+
final UploadedIndexMetadata fromXContentUploadedIndexMetadata = UploadedIndexMetadata.fromXContent(parser, 1L);
636+
assertEquals(originalUploadedIndexMetadata, fromXContentUploadedIndexMetadata);
637+
}
638+
}
639+
622640
private UploadedIndexMetadata randomlyChangingUploadedIndexMetadata(UploadedIndexMetadata uploadedIndexMetadata) {
623641
switch (randomInt(2)) {
624642
case 0:
@@ -642,4 +660,5 @@ private UploadedIndexMetadata randomlyChangingUploadedIndexMetadata(UploadedInde
642660
}
643661
return uploadedIndexMetadata;
644662
}
663+
645664
}

‎server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2254,13 +2254,14 @@ public void testReadLatestMetadataManifestSuccessButIndexMetadataFetchIOExceptio
22542254
.stateVersion(1L)
22552255
.stateUUID("state-uuid")
22562256
.clusterUUID("cluster-uuid")
2257+
.codecVersion(CODEC_V2)
22572258
.nodeId("nodeA")
22582259
.opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
22592260
.previousClusterUUID("prev-cluster-uuid")
22602261
.build();
22612262

22622263
BlobContainer blobContainer = mockBlobStoreObjects();
2263-
mockBlobContainer(blobContainer, expectedManifest, Map.of());
2264+
mockBlobContainer(blobContainer, expectedManifest, Map.of(), CODEC_V2);
22642265
when(blobContainer.readBlob(uploadedIndexMetadata.getUploadedFilename())).thenThrow(FileNotFoundException.class);
22652266

22662267
remoteClusterStateService.start();
@@ -2288,11 +2289,11 @@ public void testReadLatestMetadataManifestSuccess() throws IOException {
22882289
.clusterUUID("cluster-uuid")
22892290
.nodeId("nodeA")
22902291
.opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
2291-
.codecVersion(ClusterMetadataManifest.CODEC_V0)
2292+
.codecVersion(CODEC_V2)
22922293
.previousClusterUUID("prev-cluster-uuid")
22932294
.build();
22942295

2295-
mockBlobContainer(mockBlobStoreObjects(), expectedManifest, new HashMap<>());
2296+
mockBlobContainer(mockBlobStoreObjects(), expectedManifest, new HashMap<>(), CODEC_V2);
22962297
remoteClusterStateService.start();
22972298
final ClusterMetadataManifest manifest = remoteClusterStateService.getLatestClusterMetadataManifest(
22982299
clusterState.getClusterName().value(),
@@ -2416,10 +2417,10 @@ public void testReadLatestIndexMetadataSuccess() throws IOException {
24162417
.nodeId("nodeA")
24172418
.opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
24182419
.previousClusterUUID("prev-cluster-uuid")
2419-
.codecVersion(ClusterMetadataManifest.CODEC_V0)
2420+
.codecVersion(CODEC_V2)
24202421
.build();
24212422

2422-
mockBlobContainer(mockBlobStoreObjects(), expectedManifest, Map.of(index.getUUID(), indexMetadata));
2423+
mockBlobContainer(mockBlobStoreObjects(), expectedManifest, Map.of(index.getUUID(), indexMetadata), CODEC_V2);
24232424

24242425
Map<String, IndexMetadata> indexMetadataMap = remoteClusterStateService.getLatestClusterState(
24252426
clusterState.getClusterName().value(),
@@ -2664,6 +2665,7 @@ public void testWriteFullMetadataInParallelSuccessWithRoutingTable() throws IOEx
26642665
.clusterUUID("cluster-uuid")
26652666
.previousClusterUUID("prev-cluster-uuid")
26662667
.routingTableVersion(1)
2668+
.codecVersion(CODEC_V2)
26672669
.indicesRouting(List.of(uploadedIndiceRoutingMetadata))
26682670
.build();
26692671

@@ -3081,7 +3083,7 @@ private void mockBlobContainerForGlobalMetadata(
30813083
FORMAT_PARAMS
30823084
);
30833085
when(blobContainer.readBlob(mockManifestFileName)).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes()));
3084-
if (codecVersion >= ClusterMetadataManifest.CODEC_V2) {
3086+
if (codecVersion >= CODEC_V2) {
30853087
String coordinationFileName = getFileNameFromPath(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename());
30863088
when(blobContainer.readBlob(COORDINATION_METADATA_FORMAT.blobName(coordinationFileName))).thenAnswer((invocationOnMock) -> {
30873089
BytesReference bytesReference = COORDINATION_METADATA_FORMAT.serialize(

0 commit comments

Comments
 (0)