[DB-1640] Support Google cloud storage as archiving backend

src/Directory.Packages.props

@@ -34,6 +34,7 @@
     <PackageVersion Include="GitHubActionsTestLogger" Version="2.4.1" />
     <PackageVersion Include="GitInfo" Version="3.3.3" />
     <PackageVersion Include="Google.Api.CommonProtos" Version="2.17.0" />
+    <PackageVersion Include="Google.Cloud.Storage.V1" Version="4.13.0" />
     <PackageVersion Include="Google.Protobuf" Version="3.32.1" />
     <PackageVersion Include="Grpc.AspNetCore" Version="2.71.0" />
     <PackageVersion Include="Grpc.AspNetCore.Web" Version="2.71.0" />

src/KurrentDB.Core.XUnit.Tests/KurrentDB.Core.XUnit.Tests.csproj

@@ -3,7 +3,7 @@
 		<GenerateAssemblyInfo>true</GenerateAssemblyInfo>
 		<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
 		<!--uncomment to run various cloud-based storage tests-->
-		<!--<DefineConstants>RUN_S3_TESTS;RUN_AZ_TESTS</DefineConstants>-->
+		<!--<DefineConstants>RUN_S3_TESTS;RUN_AZ_TESTS;RUN_GCP_TESTS</DefineConstants>-->
 	</PropertyGroup>
 	<ItemGroup>
 		<PackageReference Include="GitHubActionsTestLogger" />

src/KurrentDB.Core.XUnit.Tests/Services/Archive/Storage/ArchiveStorageTestsBase.cs

@@ -20,6 +20,7 @@ namespace KurrentDB.Core.XUnit.Tests.Services.Archive.Storage;
 public abstract class ArchiveStorageTestsBase<T> : DirectoryPerTest<T> {
 	private const string AwsRegion = "eu-west-1";
 	private const string AwsBucket = "archiver-unit-tests";
+	private const string GcpBucket = "archiver-unit-tests";
 
 	private const string ChunkPrefix = "chunk-";
 	private string ArchivePath => Path.Combine(Fixture.Directory, "archive");
@@ -44,6 +45,9 @@ protected IArchiveStorage CreateSut(StorageType storageType) {
 					Region = AwsRegion,
 				},
 				Azure = AzuriteHelpers.Options,
+				GCP = new () {
+					Bucket = GcpBucket,
+				},
 			},
 			archiveNamingStrategy);
 

src/KurrentDB.Core.XUnit.Tests/Services/Archive/Storage/BlobStorageTests.cs

@@ -10,6 +10,7 @@
 using KurrentDB.Core.Services.Archive;
 using KurrentDB.Core.Services.Archive.Storage;
 using KurrentDB.Core.Services.Archive.Storage.Azure;
+using KurrentDB.Core.Services.Archive.Storage.Gcp;
 using KurrentDB.Core.Services.Archive.Storage.S3;
 using Xunit;
 
@@ -20,6 +21,7 @@ namespace KurrentDB.Core.XUnit.Tests.Services.Archive.Storage;
 public class BlobStorageTests : DirectoryPerTest<BlobStorageTests> {
 	private const string AwsRegion = "eu-west-1";
 	private const string AwsBucket = "archiver-unit-tests";
+	private const string GcpBucket = "archiver-unit-tests";
 
 	private string ArchivePath => Path.Combine(Fixture.Directory, "archive");
 	private string LocalPath => Path.Combine(Fixture.Directory, "local");
@@ -47,6 +49,11 @@ IBlobStorage CreateSut(StorageType storageType) {
 				storage = new AzureBlobStorage(AzuriteHelpers.Options);
 				AzuriteHelpers.ConfigureEnvironment();
 				break;
+			case StorageType.GCP:
+                storage = new GcpBlobStorage(new() {
+                    Bucket = GcpBucket,
+                });
+				break;
 			default:
 				throw new NotImplementedException();
 		}
@@ -68,6 +75,7 @@ private async ValueTask<FileStream> CreateFile(string fileName, int fileSize) {
 	[Theory]
 	[StorageData.S3]
 	[StorageData.Azure]
+	[StorageData.GCP]
 	[StorageData.FileSystem]
 	public async Task can_read_file_entirely(StorageType storageType) {
 		var sut = CreateSut(storageType);
@@ -85,17 +93,19 @@ public async Task can_read_file_entirely(StorageType storageType) {
 
 		// read the uploaded file
 		using var buffer = Memory.AllocateExactly<byte>(fileSize);
-		await sut.ReadAsync("output.file", buffer.Memory, offset: 0, CancellationToken.None);
+		var numRead = await sut.ReadAsync("output.file", buffer.Memory, offset: 0, CancellationToken.None);
 
 		// then
 		Assert.Equal(localContent, buffer.Span);
+		Assert.Equal(localContent.Length, numRead);
 	}
 
 	[Theory]
 	[StorageData.S3]
 	[StorageData.Azure]
+	[StorageData.GCP]
 	[StorageData.FileSystem]
-	public async Task can_store_and_read_file_partially(StorageType storageType) {
+	public async Task can_read_file_partially(StorageType storageType) {
 		var sut = CreateSut(storageType);
 
 		// create a file and upload it
@@ -113,15 +123,72 @@ public async Task can_store_and_read_file_partially(StorageType storageType) {
 		var end = localContent.Length;
 		var length = end - start;
 		using var buffer = Memory.AllocateExactly<byte>(length);
-		await sut.ReadAsync("output.file", buffer.Memory, offset: start, CancellationToken.None);
+		var numRead = await sut.ReadAsync("output.file", buffer.Memory, offset: start, CancellationToken.None);
 
 		// then
 		Assert.Equal(localContent.AsSpan(start..end), buffer.Span);
+		Assert.Equal(localContent.Length / 2, numRead);
+	}
+
+	[Theory]
+	[StorageData.S3]
+	[StorageData.Azure]
+	[StorageData.GCP]
+	[StorageData.FileSystem]
+	public async Task can_read_file_partially_and_past_end_of_file(StorageType storageType) {
+		var sut = CreateSut(storageType);
+
+		// create a file and upload it
+		string localPath;
+		await using (var fs = await CreateFile("local.file", fileSize: 1024)) {
+			await sut.StoreAsync(fs, "output.file", CancellationToken.None);
+			localPath = fs.Name;
+		}
+
+		// read the local file
+		var localContent = await File.ReadAllBytesAsync(localPath);
+
+		// read the uploaded file partially with a buffer that goes past the end of the file
+		var start = localContent.Length / 2;
+		using var buffer = Memory.AllocateExactly<byte>(localContent.Length);
+		var numRead = await sut.ReadAsync("output.file", buffer.Memory, offset: start, CancellationToken.None);
+
+		// then
+		Assert.Equal(localContent.AsSpan(start..), buffer.Span[..numRead]);
+		Assert.Equal(localContent.Length / 2, numRead);
+	}
+
+	[Theory]
+	[StorageData.S3]
+	[StorageData.Azure]
+	[StorageData.GCP]
+	[StorageData.FileSystem]
+	public async Task can_read_past_end_of_file(StorageType storageType) {
+		var sut = CreateSut(storageType);
+
+		// create a file and upload it
+		string localPath;
+		await using (var fs = await CreateFile("local.file", fileSize: 1024)) {
+			await sut.StoreAsync(fs, "output.file", CancellationToken.None);
+			localPath = fs.Name;
+		}
+
+		// read the local file
+		var localContent = await File.ReadAllBytesAsync(localPath);
+
+		// read past the end of the uploaded file
+		var start = localContent.Length;
+		using var buffer = Memory.AllocateExactly<byte>(localContent.Length);
+		var numRead = await sut.ReadAsync("output.file", buffer.Memory, offset: start, CancellationToken.None);
+
+		// then
+		Assert.Equal(0, numRead);
 	}
 
 	[Theory]
 	[StorageData.S3]
 	[StorageData.Azure]
+	[StorageData.GCP]
 	[StorageData.FileSystem]
 	public async Task can_retrieve_metadata(StorageType storageType) {
 		var sut = CreateSut(storageType);
@@ -142,6 +209,7 @@ public async Task can_retrieve_metadata(StorageType storageType) {
 	[Theory]
 	[StorageData.S3]
 	[StorageData.Azure]
+	[StorageData.GCP]
 	[StorageData.FileSystem]
 	public async Task read_missing_file_throws_FileNotFoundException(StorageType storageType) {
 		var sut = CreateSut(storageType);

src/KurrentDB.Core.XUnit.Tests/Services/Archive/Storage/GcpCliDirectoryNotFoundException.cs

@@ -0,0 +1,9 @@
+// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements.
+// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md).
+
+using System.IO;
+
+namespace KurrentDB.Core.XUnit.Tests.Services.Archive.Storage;
+
+public sealed class GcpCliDirectoryNotFoundException(string path)
+	: DirectoryNotFoundException($"Directory '{path}' with config files for Google Cloud CLI doesn't exist");

src/KurrentDB.Core.XUnit.Tests/Services/Archive/Storage/RemoteFileSystemTests.cs

@@ -25,6 +25,7 @@ public sealed class RemoteFileSystemTests : ArchiveStorageTestsBase<RemoteFileSy
 	[Theory]
 	[StorageData.S3]
 	[StorageData.Azure]
+	[StorageData.GCP]
 	[StorageData.FileSystem]
 	public async Task chunk_can_read_record_from_object_storage(StorageType storageType) {
 		const int recordsCount = 10;
@@ -72,6 +73,7 @@ public async Task chunk_can_read_record_from_object_storage(StorageType storageT
 	[Theory]
 	[StorageData.S3]
 	[StorageData.Azure]
+	[StorageData.GCP]
 	[StorageData.FileSystem]
 	public async Task chunk_can_bulk_read_record_from_object_storage(StorageType storageType) {
 		const int recordsCount = 10;

src/KurrentDB.Core.XUnit.Tests/Services/Archive/Storage/RemoteStorageTheoryAttribute.cs

@@ -56,6 +56,32 @@ private static void CheckPrerequisites(ref bool symbolSet) {
 		}
 	}
 
+	public sealed class GCPAttribute(params object[] args) : RemoteStorageDataAttribute(
+		StorageType.GCP,
+		args,
+		Symbol,
+		static (ref bool isSet) => CheckPrerequisites(ref isSet)) {
+		const string Symbol = "RUN_GCP_TESTS";
+
+		[Conditional(Symbol)]
+		private static void CheckPrerequisites(ref bool symbolSet) {
+			symbolSet = true;
+			const string gcpDirectoryNameLinux = ".config/gcloud";
+			const string gcpDirectoryNameWindows = "gcloud";
+			var homeDir = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
+
+			if (OperatingSystem.IsLinux())
+				homeDir = Path.Combine(homeDir, gcpDirectoryNameLinux);
+			else if (OperatingSystem.IsWindows())
+				homeDir = Path.Combine(homeDir, gcpDirectoryNameWindows);
+			else
+				throw new NotSupportedException();
+
+			if (!Directory.Exists(homeDir))
+				throw new GcpCliDirectoryNotFoundException(homeDir);
+		}
+	}
+
 	public sealed class FileSystemAttribute(params object[] args) : RemoteStorageDataAttribute(
 		StorageType.FileSystemDevelopmentOnly,
 		args,

src/KurrentDB.Core/KurrentDB.Core.csproj

@@ -12,6 +12,7 @@
 		<PackageReference Include="FluentStorage.AWS" />
 		<PackageReference Include="Azure.Storage.Blobs" />
 		<PackageReference Include="Azure.Identity" />
+		<PackageReference Include="Google.Cloud.Storage.V1"/>
 		<PackageReference Include="Google.Protobuf" />
 		<PackageReference Include="Grpc.AspNetCore" />
 		<PackageReference Include="Grpc.Tools">

src/KurrentDB.Core/Services/Archive/ArchiveOptions.cs

@@ -12,6 +12,7 @@ public class ArchiveOptions {
 	public FileSystemOptions FileSystem { get; init; } = new();
 	public S3Options S3 { get; init; } = new();
 	public AzureOptions Azure { get; init; } = new();
+	public GcpOptions GCP { get; init; } = new();
 	public RetentionOptions RetainAtLeast { get; init; } = new();
 
 	public void Validate() {
@@ -28,7 +29,7 @@ private void ValidateImpl() {
 
 		switch (StorageType) {
 			case StorageType.Unspecified:
-				throw new InvalidConfigurationException("Please specify a StorageType (e.g. S3)");
+				throw new InvalidConfigurationException("Please specify a StorageType (e.g. S3, Azure, GCP)");
 			case StorageType.FileSystemDevelopmentOnly:
 				FileSystem.Validate();
 				break;
@@ -37,6 +38,9 @@ private void ValidateImpl() {
 				break;
 			case StorageType.Azure:
 				Azure.Validate();
+                break;
+            case StorageType.GCP:
+				GCP.Validate();
 				break;
 			default:
 				throw new InvalidConfigurationException("Unknown StorageType");
@@ -53,13 +57,14 @@ public enum StorageType {
 	FileSystemDevelopmentOnly,
 	S3,
 	Azure,
+	GCP,
 }
 
 public class FileSystemOptions {
 	public string Path { get; init; } = "";
 
 	public void Validate() {
-		if (string.IsNullOrEmpty(Path))
+		if (string.IsNullOrWhiteSpace(Path))
 			throw new InvalidConfigurationException("Please provide a Path for the FileSystem archive");
 	}
 }
@@ -69,10 +74,10 @@ public class S3Options {
 	public string Region { get; init; } = "";
 
 	public void Validate() {
-		if (string.IsNullOrEmpty(Bucket))
+		if (string.IsNullOrWhiteSpace(Bucket))
 			throw new InvalidConfigurationException("Please provide a Bucket for the S3 archive");
 
-		if (string.IsNullOrEmpty(Region))
+		if (string.IsNullOrWhiteSpace(Region))
 			throw new InvalidConfigurationException("Please provide a Region for the S3 archive");
 	}
 }
@@ -161,6 +166,15 @@ public enum AuthenticationType {
 	}
 }
 
+public class GcpOptions {
+	public string Bucket { get; init; } = "";
+
+	public void Validate() {
+		if (string.IsNullOrWhiteSpace(Bucket))
+			throw new InvalidConfigurationException("Please provide a Bucket for the GCP archive");
+	}
+}
+
 // Local chunks are removed after they have passed beyond both criteria, so they
 // must both be set to be useful.
 public class RetentionOptions {

src/KurrentDB.Core/Services/Archive/Storage/ArchiveStorageFactory.cs

@@ -4,6 +4,7 @@
 using System;
 using KurrentDB.Core.Services.Archive.Naming;
 using KurrentDB.Core.Services.Archive.Storage.Azure;
+using KurrentDB.Core.Services.Archive.Storage.Gcp;
 using KurrentDB.Core.Services.Archive.Storage.S3;
 
 namespace KurrentDB.Core.Services.Archive.Storage;
@@ -17,6 +18,7 @@ public static IArchiveStorage Create(ArchiveOptions options, IArchiveNamingStrat
 			StorageType.FileSystemDevelopmentOnly => new FileSystemBlobStorage(options.FileSystem),
 			StorageType.S3 => new S3BlobStorage(options.S3),
 			StorageType.Azure => new AzureBlobStorage(options.Azure),
+			StorageType.GCP => new GcpBlobStorage(options.GCP),
 			_ => throw new ArgumentOutOfRangeException(nameof(options.StorageType))
 		};