Skip to content

Commit de3cb8a

Browse files
ashking94ashking94
authored
[Backport 2.x] Add implementation for remote store path types (#13423)
* Add implementation for remote store path types (#13103) Signed-off-by: Ashish Singh <ssashish@amazon.com> * Fix testLocalOnlyTranslogCleanupOnNodeRestart due to code differences Signed-off-by: Ashish Singh <ssashish@amazon.com> --------- Signed-off-by: Ashish Singh <ssashish@amazon.com>
1 parent aa50a60 commit de3cb8a

File tree

14 files changed

+626
-64
lines changed

14 files changed

+626
-64
lines changed

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.opensearch.cluster.ClusterState;
2020
import org.opensearch.cluster.metadata.IndexMetadata;
2121
import org.opensearch.common.Nullable;
22+
import org.opensearch.common.blobstore.BlobPath;
2223
import org.opensearch.common.io.PathUtils;
2324
import org.opensearch.common.settings.Settings;
2425
import org.opensearch.common.util.io.IOUtils;
@@ -54,6 +55,10 @@
5455
import java.util.stream.Stream;
5556

5657
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
58+
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
59+
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
60+
import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
61+
import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
5762
import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING;
5863
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
5964
import static org.hamcrest.Matchers.equalTo;
@@ -221,6 +226,11 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
221226
String restoredIndexName1version1 = indexName1 + "-restored-1";
222227
String restoredIndexName1version2 = indexName1 + "-restored-2";
223228

229+
client(clusterManagerNode).admin()
230+
.cluster()
231+
.prepareUpdateSettings()
232+
.setTransientSettings(Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), PathType.FIXED))
233+
.get();
224234
createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true));
225235
Client client = client();
226236
Settings indexSettings = getIndexSettings(1, 0).build();
@@ -418,12 +428,15 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
418428
}
419429

420430
void assertRemoteSegmentsAndTranslogUploaded(String idx) throws IOException {
421-
String indexUUID = client().admin().indices().prepareGetSettings(idx).get().getSetting(idx, IndexMetadata.SETTING_INDEX_UUID);
422-
423-
Path remoteTranslogMetadataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/translog/metadata");
424-
Path remoteTranslogDataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/translog/data");
425-
Path segmentMetadataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/segments/metadata");
426-
Path segmentDataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/segments/data");
431+
Client client = client();
432+
String path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", TRANSLOG, METADATA).buildAsString();
433+
Path remoteTranslogMetadataPath = Path.of(remoteRepoPath + "/" + path);
434+
path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", TRANSLOG, DATA).buildAsString();
435+
Path remoteTranslogDataPath = Path.of(remoteRepoPath + "/" + path);
436+
path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", SEGMENTS, METADATA).buildAsString();
437+
Path segmentMetadataPath = Path.of(remoteRepoPath + "/" + path);
438+
path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", SEGMENTS, DATA).buildAsString();
439+
Path segmentDataPath = Path.of(remoteRepoPath + "/" + path);
427440

428441
try (
429442
Stream<Path> translogMetadata = Files.list(remoteTranslogMetadataPath);

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java

Lines changed: 20 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.opensearch.cluster.routing.RecoverySource;
2424
import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand;
2525
import org.opensearch.common.Priority;
26+
import org.opensearch.common.blobstore.BlobPath;
2627
import org.opensearch.common.settings.Settings;
2728
import org.opensearch.common.unit.TimeValue;
2829
import org.opensearch.common.util.concurrent.BufferedAsyncIOProcessor;
@@ -58,7 +59,12 @@
5859

5960
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
6061
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
62+
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
63+
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
64+
import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
65+
import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
6166
import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
67+
import static org.opensearch.test.OpenSearchTestCase.getShardLevelBlobPath;
6268
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
6369
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
6470
import static org.hamcrest.Matchers.comparesEqualTo;
@@ -183,13 +189,9 @@ public void testStaleCommitDeletionWithInvokeFlush() throws Exception {
183189
createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1));
184190
int numberOfIterations = randomIntBetween(5, 15);
185191
indexData(numberOfIterations, true, INDEX_NAME);
186-
String indexUUID = client().admin()
187-
.indices()
188-
.prepareGetSettings(INDEX_NAME)
189-
.get()
190-
.getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID);
191-
Path indexPath = Path.of(String.valueOf(segmentRepoPath), indexUUID, "/0/segments/metadata");
192-
192+
String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, METADATA).buildAsString();
193+
Path indexPath = Path.of(segmentRepoPath + "/" + shardPath);
194+
;
193195
IndexShard indexShard = getIndexShard(dataNode, INDEX_NAME);
194196
int lastNMetadataFilesToKeep = indexShard.getRemoteStoreSettings().getMinRemoteSegmentMetadataFiles();
195197
// Delete is async.
@@ -213,12 +215,8 @@ public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception {
213215
createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1));
214216
int numberOfIterations = randomIntBetween(5, 15);
215217
indexData(numberOfIterations, false, INDEX_NAME);
216-
String indexUUID = client().admin()
217-
.indices()
218-
.prepareGetSettings(INDEX_NAME)
219-
.get()
220-
.getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID);
221-
Path indexPath = Path.of(String.valueOf(segmentRepoPath), indexUUID, "/0/segments/metadata");
218+
String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, METADATA).buildAsString();
219+
Path indexPath = Path.of(segmentRepoPath + "/" + shardPath);
222220
int actualFileCount = getFileCount(indexPath);
223221
// We also allow (numberOfIterations + 1) as index creation also triggers refresh.
224222
MatcherAssert.assertThat(actualFileCount, is(oneOf(numberOfIterations - 1, numberOfIterations, numberOfIterations + 1)));
@@ -232,12 +230,8 @@ public void testStaleCommitDeletionWithMinSegmentFiles_3() throws Exception {
232230
createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1));
233231
int numberOfIterations = randomIntBetween(5, 15);
234232
indexData(numberOfIterations, true, INDEX_NAME);
235-
String indexUUID = client().admin()
236-
.indices()
237-
.prepareGetSettings(INDEX_NAME)
238-
.get()
239-
.getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID);
240-
Path indexPath = Path.of(String.valueOf(segmentRepoPath), indexUUID, "/0/segments/metadata");
233+
String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, METADATA).buildAsString();
234+
Path indexPath = Path.of(segmentRepoPath + "/" + shardPath);
241235
int actualFileCount = getFileCount(indexPath);
242236
// We also allow (numberOfIterations + 1) as index creation also triggers refresh.
243237
MatcherAssert.assertThat(actualFileCount, is(oneOf(4)));
@@ -251,12 +245,9 @@ public void testStaleCommitDeletionWithMinSegmentFiles_Disabled() throws Excepti
251245
createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1));
252246
int numberOfIterations = randomIntBetween(12, 18);
253247
indexData(numberOfIterations, true, INDEX_NAME);
254-
String indexUUID = client().admin()
255-
.indices()
256-
.prepareGetSettings(INDEX_NAME)
257-
.get()
258-
.getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID);
259-
Path indexPath = Path.of(String.valueOf(segmentRepoPath), indexUUID, "/0/segments/metadata");
248+
String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, METADATA).buildAsString();
249+
Path indexPath = Path.of(segmentRepoPath + "/" + shardPath);
250+
;
260251
int actualFileCount = getFileCount(indexPath);
261252
// We also allow (numberOfIterations + 1) as index creation also triggers refresh.
262253
MatcherAssert.assertThat(actualFileCount, is(oneOf(numberOfIterations + 1)));
@@ -590,12 +581,8 @@ public void testFallbackToNodeToNodeSegmentCopy() throws Exception {
590581
flushAndRefresh(INDEX_NAME);
591582

592583
// 3. Delete data from remote segment store
593-
String indexUUID = client().admin()
594-
.indices()
595-
.prepareGetSettings(INDEX_NAME)
596-
.get()
597-
.getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID);
598-
Path segmentDataPath = Path.of(String.valueOf(segmentRepoPath), indexUUID, "/0/segments/data");
584+
String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, DATA).buildAsString();
585+
Path segmentDataPath = Path.of(segmentRepoPath + "/" + shardPath);
599586

600587
try (Stream<Path> files = Files.list(segmentDataPath)) {
601588
files.forEach(p -> {
@@ -850,7 +837,8 @@ public void testLocalOnlyTranslogCleanupOnNodeRestart() throws Exception {
850837
.get()
851838
.getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID);
852839

853-
Path translogMetaDataPath = Path.of(String.valueOf(translogRepoPath), indexUUID, "/0/translog/metadata");
840+
String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", TRANSLOG, METADATA).buildAsString();
841+
Path translogMetaDataPath = Path.of(translogRepoPath + "/" + shardPath);
854842

855843
try (Stream<Path> files = Files.list(translogMetaDataPath)) {
856844
files.forEach(p -> {

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
1212
import org.opensearch.action.admin.indices.stats.IndicesStatsRequest;
1313
import org.opensearch.action.admin.indices.stats.IndicesStatsResponse;
14+
import org.opensearch.common.blobstore.BlobPath;
1415
import org.opensearch.common.settings.Settings;
1516
import org.opensearch.test.OpenSearchIntegTestCase;
1617

@@ -22,7 +23,10 @@
2223
import java.util.Set;
2324
import java.util.concurrent.TimeUnit;
2425

26+
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
27+
import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
2528
import static org.opensearch.index.remote.RemoteStorePressureSettings.REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED;
29+
import static org.opensearch.test.OpenSearchTestCase.getShardLevelBlobPath;
2630

2731
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
2832
public class RemoteStoreRefreshListenerIT extends AbstractRemoteStoreMockRepositoryIntegTestCase {
@@ -45,8 +49,10 @@ public void testRemoteRefreshRetryOnFailure() throws Exception {
4549
IndicesStatsResponse response = client().admin().indices().stats(new IndicesStatsRequest()).get();
4650
assertEquals(1, response.getShards().length);
4751

52+
String indexName = response.getShards()[0].getShardRouting().index().getName();
4853
String indexUuid = response.getShards()[0].getShardRouting().index().getUUID();
49-
Path segmentDataRepoPath = location.resolve(String.format(Locale.ROOT, "%s/0/segments/data", indexUuid));
54+
String shardPath = getShardLevelBlobPath(client(), indexName, new BlobPath(), "0", SEGMENTS, DATA).buildAsString();
55+
Path segmentDataRepoPath = location.resolve(shardPath);
5056
String segmentDataLocalPath = String.format(Locale.ROOT, "%s/indices/%s/0/index", response.getShards()[0].getDataPath(), indexUuid);
5157

5258
logger.info("--> Verify that the segment files are same on local and repository eventually");

‎server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,14 @@
1414
import org.opensearch.cluster.metadata.IndexMetadata;
1515
import org.opensearch.common.UUIDs;
1616
import org.opensearch.common.action.ActionFuture;
17+
import org.opensearch.common.blobstore.BlobContainer;
18+
import org.opensearch.common.blobstore.BlobPath;
1719
import org.opensearch.common.settings.Settings;
1820
import org.opensearch.common.unit.TimeValue;
21+
import org.opensearch.index.store.RemoteBufferedOutputDirectory;
1922
import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
23+
import org.opensearch.repositories.RepositoriesService;
24+
import org.opensearch.repositories.blobstore.BlobStoreRepository;
2025
import org.opensearch.test.OpenSearchIntegTestCase;
2126

2227
import java.nio.file.Path;
@@ -27,6 +32,8 @@
2732
import java.util.concurrent.TimeUnit;
2833
import java.util.stream.Stream;
2934

35+
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
36+
import static org.opensearch.index.remote.RemoteStoreEnums.DataType.LOCK_FILES;
3037
import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings;
3138
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
3239
import static org.hamcrest.Matchers.comparesEqualTo;
@@ -308,7 +315,21 @@ public void testRemoteStoreCleanupForDeletedIndex() throws Exception {
308315
SnapshotInfo snapshotInfo1 = createFullSnapshot(snapshotRepoName, "snap1");
309316
SnapshotInfo snapshotInfo2 = createFullSnapshot(snapshotRepoName, "snap2");
310317

311-
String[] lockFiles = getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME);
318+
final RepositoriesService repositoriesService = internalCluster().getCurrentClusterManagerNodeInstance(RepositoriesService.class);
319+
final BlobStoreRepository remoteStoreRepository = (BlobStoreRepository) repositoriesService.repository(REMOTE_REPO_NAME);
320+
BlobPath shardLevelBlobPath = getShardLevelBlobPath(
321+
client(),
322+
remoteStoreEnabledIndexName,
323+
remoteStoreRepository.basePath(),
324+
"0",
325+
SEGMENTS,
326+
LOCK_FILES
327+
);
328+
BlobContainer blobContainer = remoteStoreRepository.blobStore().blobContainer(shardLevelBlobPath);
329+
String[] lockFiles;
330+
try (RemoteBufferedOutputDirectory lockDirectory = new RemoteBufferedOutputDirectory(blobContainer)) {
331+
lockFiles = lockDirectory.listAll();
332+
}
312333
assert (lockFiles.length == 2) : "lock files are " + Arrays.toString(lockFiles);
313334

314335
// delete the giremote store index
@@ -321,7 +342,9 @@ public void testRemoteStoreCleanupForDeletedIndex() throws Exception {
321342
.get();
322343
assertAcked(deleteSnapshotResponse);
323344

324-
lockFiles = getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME, indexUUID);
345+
try (RemoteBufferedOutputDirectory lockDirectory = new RemoteBufferedOutputDirectory(blobContainer)) {
346+
lockFiles = lockDirectory.listAll();
347+
}
325348
assert (lockFiles.length == 1) : "lock files are " + Arrays.toString(lockFiles);
326349
assertTrue(lockFiles[0].contains(snapshotInfo2.snapshotId().getUUID()));
327350

‎server/src/main/java/org/opensearch/common/blobstore/BlobPath.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,15 @@ public BlobPath add(String path) {
7979
return new BlobPath(Collections.unmodifiableList(paths));
8080
}
8181

82+
/**
83+
* Add additional level of paths to the existing path and returns new {@link BlobPath} with the updated paths.
84+
*/
85+
public BlobPath add(Iterable<String> paths) {
86+
List<String> updatedPaths = new ArrayList<>(this.paths);
87+
paths.iterator().forEachRemaining(updatedPaths::add);
88+
return new BlobPath(Collections.unmodifiableList(updatedPaths));
89+
}
90+
8291
public String buildAsString() {
8392
String p = String.join(SEPARATOR, paths);
8493
if (p.isEmpty() || p.endsWith(SEPARATOR)) {

‎server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -103,9 +103,27 @@ boolean requiresHashAlgorithm() {
103103
HASHED_PREFIX(1) {
104104
@Override
105105
public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
106-
// TODO - We need to implement this, keeping the same path as Fixed for sake of multiple tests that can fail otherwise.
107-
// throw new UnsupportedOperationException("Not implemented"); --> Not using this for unblocking couple of tests.
106+
assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null";
107+
return BlobPath.cleanPath()
108+
.add(hashAlgorithm.hash(pathInput))
109+
.add(pathInput.basePath())
110+
.add(pathInput.indexUUID())
111+
.add(pathInput.shardId())
112+
.add(pathInput.dataCategory().getName())
113+
.add(pathInput.dataType().getName());
114+
}
115+
116+
@Override
117+
boolean requiresHashAlgorithm() {
118+
return true;
119+
}
120+
},
121+
HASHED_INFIX(2) {
122+
@Override
123+
public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
124+
assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null";
108125
return pathInput.basePath()
126+
.add(hashAlgorithm.hash(pathInput))
109127
.add(pathInput.indexUUID())
110128
.add(pathInput.shardId())
111129
.add(pathInput.dataCategory().getName())
@@ -200,10 +218,11 @@ public enum PathHashAlgorithm {
200218

201219
FNV_1A(0) {
202220
@Override
203-
long hash(PathInput pathInput) {
221+
String hash(PathInput pathInput) {
204222
String input = pathInput.indexUUID() + pathInput.shardId() + pathInput.dataCategory().getName() + pathInput.dataType()
205223
.getName();
206-
return FNV1a.hash32(input);
224+
long hash = FNV1a.hash64(input);
225+
return RemoteStoreUtils.longToUrlBase64(hash);
207226
}
208227
};
209228

@@ -218,6 +237,7 @@ public int getCode() {
218237
}
219238

220239
private static final Map<Integer, PathHashAlgorithm> CODE_TO_ENUM;
240+
221241
static {
222242
PathHashAlgorithm[] values = values();
223243
Map<Integer, PathHashAlgorithm> codeToStatus = new HashMap<>(values.length);
@@ -240,7 +260,7 @@ public static PathHashAlgorithm fromCode(int code) {
240260
return CODE_TO_ENUM.get(code);
241261
}
242262

243-
abstract long hash(PathInput pathInput);
263+
abstract String hash(PathInput pathInput);
244264

245265
public static PathHashAlgorithm parseString(String pathHashAlgorithm) {
246266
try {

‎server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@
1010

1111
import org.opensearch.common.collect.Tuple;
1212

13+
import java.nio.ByteBuffer;
1314
import java.util.Arrays;
15+
import java.util.Base64;
1416
import java.util.HashMap;
1517
import java.util.List;
1618
import java.util.Map;
@@ -101,4 +103,17 @@ public static void verifyNoMultipleWriters(List<String> mdFiles, Function<String
101103
});
102104
}
103105

106+
/**
107+
* Converts an input hash which occupies 64 bits of space into Base64 (6 bits per character) String. This must not
108+
* be changed as it is used for creating path for storing remote store data on the remote store.
109+
* This converts the byte array to base 64 string. `/` is replaced with `_`, `+` is replaced with `-` and `=`
110+
* which is padded at the last is also removed. These characters are either used as delimiter or special character
111+
* requiring special handling in some vendors. The characters present in this base64 version are [A-Za-z0-9_-].
112+
* This must not be changed as it is used for creating path for storing remote store data on the remote store.
113+
*/
114+
static String longToUrlBase64(long value) {
115+
byte[] hashBytes = ByteBuffer.allocate(Long.BYTES).putLong(value).array();
116+
String base64Str = Base64.getUrlEncoder().encodeToString(hashBytes);
117+
return base64Str.substring(0, base64Str.length() - 1);
118+
}
104119
}

0 commit comments

Comments
 (0)