From b4f02777f1171c2055c00fede675f41ca86eb87e Mon Sep 17 00:00:00 2001 From: John Simons Date: Tue, 26 Aug 2025 11:35:42 +1000 Subject: [PATCH 01/41] Added a Postgres persister for Audit --- src/ProjectReferences.Persisters.Audit.props | 1 + .../PostgresqlAttachmentsBodyStorage.cs | 13 +++++++++ .../PostgresqlAuditDataStore.cs | 26 +++++++++++++++++ .../PostgresqlFailedAuditStorage.cs | 15 ++++++++++ .../PostgresqlPersistence.cs | 11 +++++++ .../PostgresqlPersistenceConfiguration.cs | 14 +++++++++ ...ontrol.Audit.Persistence.Postgresql.csproj | 19 ++++++++++++ .../PostgresqlAuditIngestionUnitOfWork.cs | 29 +++++++++++++++++++ ...stgresqlAuditIngestionUnitOfWorkFactory.cs | 17 +++++++++++ .../persistence.manifest | 7 +++++ src/ServiceControl.sln | 16 ++++++++++ 11 files changed, 168 insertions(+) create mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/BodyStorage/PostgresqlAttachmentsBodyStorage.cs create mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlAuditDataStore.cs create mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlFailedAuditStorage.cs create mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistence.cs create mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistenceConfiguration.cs create mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/ServiceControl.Audit.Persistence.Postgresql.csproj create mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWork.cs create mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWorkFactory.cs create mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/persistence.manifest diff --git a/src/ProjectReferences.Persisters.Audit.props b/src/ProjectReferences.Persisters.Audit.props index 0a9c4d0dcb..3edbbf5498 100644 --- a/src/ProjectReferences.Persisters.Audit.props +++ b/src/ProjectReferences.Persisters.Audit.props @@ -3,6 +3,7 @@ + \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/BodyStorage/PostgresqlAttachmentsBodyStorage.cs b/src/ServiceControl.Audit.Persistence.Postgresql/BodyStorage/PostgresqlAttachmentsBodyStorage.cs new file mode 100644 index 0000000000..2ade14d8d7 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Postgresql/BodyStorage/PostgresqlAttachmentsBodyStorage.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Audit.Persistence.Postgresql.BodyStorage +{ + using System.IO; + using System.Threading; + using System.Threading.Tasks; + using ServiceControl.Audit.Auditing.BodyStorage; + + public class PostgresqlAttachmentsBodyStorage : IBodyStorage + { + public Task Store(string bodyId, string contentType, int bodySize, Stream bodyStream, CancellationToken cancellationToken) => throw new System.NotImplementedException(); + public Task TryFetch(string bodyId, CancellationToken cancellationToken) => throw new System.NotImplementedException(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlAuditDataStore.cs new file mode 100644 index 0000000000..fdf8e99d72 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlAuditDataStore.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Audit.Persistence.Postgresql +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using ServiceControl.Audit.Auditing; + using ServiceControl.Audit.Auditing.MessagesView; + using ServiceControl.Audit.Infrastructure; + using ServiceControl.Audit.Monitoring; + using ServiceControl.Audit.Persistence; + using ServiceControl.SagaAudit; + + public class PostgresqlAuditDataStore : IAuditDataStore + { + public Task GetMessageBody(string messageId, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task>> QueryKnownEndpoints(CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task> QuerySagaHistoryById(Guid input, CancellationToken cancellationToken) => throw new NotImplementedException(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlFailedAuditStorage.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlFailedAuditStorage.cs new file mode 100644 index 0000000000..7e1b7402f5 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlFailedAuditStorage.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Audit.Persistence.Postgresql +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using ServiceControl.Audit.Auditing; + using ServiceControl.Audit.Persistence; + + public class PostgresqlFailedAuditStorage : IFailedAuditStorage + { + public Task GetFailedAuditsCount() => throw new NotImplementedException(); + public Task ProcessFailedMessages(Func, CancellationToken, Task> onMessage, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task SaveFailedAuditImport(FailedAuditImport message) => throw new NotImplementedException(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistence.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistence.cs new file mode 100644 index 0000000000..4325a528c6 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistence.cs @@ -0,0 +1,11 @@ +namespace ServiceControl.Audit.Persistence.Postgresql +{ + using Microsoft.Extensions.DependencyInjection; + using ServiceControl.Audit.Persistence; + + public class PostgresqlPersistence : IPersistence + { + public void AddInstaller(IServiceCollection services) => throw new System.NotImplementedException(); + public void AddPersistence(IServiceCollection services) => throw new System.NotImplementedException(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistenceConfiguration.cs new file mode 100644 index 0000000000..12091cc160 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistenceConfiguration.cs @@ -0,0 +1,14 @@ +namespace ServiceControl.Audit.Persistence.Postgresql +{ + using System.Collections.Generic; + using ServiceControl.Audit.Persistence; + + public class PostgresqlPersistenceConfiguration : IPersistenceConfiguration + { + public string Name => "PostgreSQL"; + + public IEnumerable ConfigurationKeys => new[] { "ConnectionString", "MaxConnections" }; + + public IPersistence Create(PersistenceSettings settings) => throw new System.NotImplementedException(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/ServiceControl.Audit.Persistence.Postgresql.csproj b/src/ServiceControl.Audit.Persistence.Postgresql/ServiceControl.Audit.Persistence.Postgresql.csproj new file mode 100644 index 0000000000..b2232a9098 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Postgresql/ServiceControl.Audit.Persistence.Postgresql.csproj @@ -0,0 +1,19 @@ + + + + net8.0 + true + true + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWork.cs new file mode 100644 index 0000000000..d1fd17f3eb --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWork.cs @@ -0,0 +1,29 @@ +namespace ServiceControl.Audit.Persistence.Postgresql.UnitOfWork +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using ServiceControl.Audit.Auditing; + using ServiceControl.Audit.Persistence.Monitoring; + using ServiceControl.Audit.Persistence.UnitOfWork; + using ServiceControl.SagaAudit; + + public class PostgresqlAuditIngestionUnitOfWork : IAuditIngestionUnitOfWork + { + public ValueTask DisposeAsync() + { + // TODO: Dispose resources if needed + return ValueTask.CompletedTask; + } + + public Task CompleteAsync(CancellationToken cancellationToken = default) + { + // TODO: Commit transaction or batch + throw new NotImplementedException(); + } + + public Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body = default, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWorkFactory.cs new file mode 100644 index 0000000000..e80fcee552 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWorkFactory.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Audit.Persistence.Postgresql.UnitOfWork +{ + using System.Threading; + using System.Threading.Tasks; + using ServiceControl.Audit.Persistence.UnitOfWork; + + public class PostgresqlAuditIngestionUnitOfWorkFactory : IAuditIngestionUnitOfWorkFactory + { + public ValueTask StartNew(int batchSize, CancellationToken cancellationToken) + { + // TODO: Implement logic to start a new unit of work for PostgreSQL + throw new System.NotImplementedException(); + } + + public bool CanIngestMore() => true; // TODO: Implement logic based on storage state + } +} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/persistence.manifest b/src/ServiceControl.Audit.Persistence.Postgresql/persistence.manifest new file mode 100644 index 0000000000..b1b970370c --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Postgresql/persistence.manifest @@ -0,0 +1,7 @@ +{ + "Name": "Postgresql", + "DisplayName": "Postgresql", + "Description": "Postgresql ServiceControl Audit persister", + "AssemblyName": "ServiceControl.Audit.Persistence.Postgresql", + "TypeName": "ServiceControl.Audit.Persistence.Postgresql.PostgresqlPersistenceConfiguration, ServiceControl.Audit.Persistence.Postgresql" +} \ No newline at end of file diff --git a/src/ServiceControl.sln b/src/ServiceControl.sln index fa8d9a30e6..19e85db8c0 100644 --- a/src/ServiceControl.sln +++ b/src/ServiceControl.sln @@ -1,3 +1,4 @@ + Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.0.31815.197 @@ -187,6 +188,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceControl.Hosting", "S EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SetupProcessFake", "SetupProcessFake\SetupProcessFake.csproj", "{5837F789-69B9-44BE-B114-3A2880F06CAB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceControl.Audit.Persistence.Postgresql", "ServiceControl.Audit.Persistence.Postgresql\ServiceControl.Audit.Persistence.Postgresql.csproj", "{AD3CA060-83E5-4E39-82A3-774023B057DF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1025,6 +1028,18 @@ Global {5837F789-69B9-44BE-B114-3A2880F06CAB}.Release|x64.Build.0 = Release|Any CPU {5837F789-69B9-44BE-B114-3A2880F06CAB}.Release|x86.ActiveCfg = Release|Any CPU {5837F789-69B9-44BE-B114-3A2880F06CAB}.Release|x86.Build.0 = Release|Any CPU + {AD3CA060-83E5-4E39-82A3-774023B057DF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AD3CA060-83E5-4E39-82A3-774023B057DF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AD3CA060-83E5-4E39-82A3-774023B057DF}.Debug|x64.ActiveCfg = Debug|Any CPU + {AD3CA060-83E5-4E39-82A3-774023B057DF}.Debug|x64.Build.0 = Debug|Any CPU + {AD3CA060-83E5-4E39-82A3-774023B057DF}.Debug|x86.ActiveCfg = Debug|Any CPU + {AD3CA060-83E5-4E39-82A3-774023B057DF}.Debug|x86.Build.0 = Debug|Any CPU + {AD3CA060-83E5-4E39-82A3-774023B057DF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AD3CA060-83E5-4E39-82A3-774023B057DF}.Release|Any CPU.Build.0 = Release|Any CPU + {AD3CA060-83E5-4E39-82A3-774023B057DF}.Release|x64.ActiveCfg = Release|Any CPU + {AD3CA060-83E5-4E39-82A3-774023B057DF}.Release|x64.Build.0 = Release|Any CPU + {AD3CA060-83E5-4E39-82A3-774023B057DF}.Release|x86.ActiveCfg = Release|Any CPU + {AD3CA060-83E5-4E39-82A3-774023B057DF}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1110,6 +1125,7 @@ Global {18DBEEF5-42EE-4C1D-A05B-87B21C067D53} = {E0E45F22-35E3-4AD8-B09E-EFEA5A2F18EE} {481032A1-1106-4C6C-B75E-512F2FB08882} = {9AF9D3C7-E859-451B-BA4D-B954D289213A} {5837F789-69B9-44BE-B114-3A2880F06CAB} = {927A078A-E271-4878-A153-86D71AE510E2} + {AD3CA060-83E5-4E39-82A3-774023B057DF} = {BD162BC6-705F-45B4-A6B5-C138DC966C1D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3B9E5B72-F580-465A-A22C-2D2148AF4EB4} From 22a5bcd7fb812e49eecf55e8d4ef70da4b1393ea Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 27 Aug 2025 09:18:55 +1000 Subject: [PATCH 02/41] Renaming to uppercase SQL --- src/Directory.Packages.props | 1 + src/ProjectReferences.Persisters.Audit.props | 2 +- .../PostgresqlAttachmentsBodyStorage.cs | 4 +- .../PostgreSQLConnectionFactory.cs | 28 +++++ .../PostgresqlAuditDataStore.cs | 4 +- .../PostgresqlFailedAuditStorage.cs | 4 +- .../PostgresqlPersistence.cs | 23 +++- .../PostgresqlPersistenceConfiguration.cs | 76 +++++++++++- ...ontrol.Audit.Persistence.Postgresql.csproj | 11 +- .../PostgresqlAuditIngestionUnitOfWork.cs | 117 ++++++++++++++++-- ...stgresqlAuditIngestionUnitOfWorkFactory.cs | 20 ++- .../persistence.manifest | 10 +- .../DevelopmentPersistenceLocations.cs | 1 + src/ServiceControl.sln | 2 +- 14 files changed, 265 insertions(+), 38 deletions(-) create mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/PostgreSQLConnectionFactory.cs diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 7420a4033f..f8b8956d52 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -32,6 +32,7 @@ + diff --git a/src/ProjectReferences.Persisters.Audit.props b/src/ProjectReferences.Persisters.Audit.props index 3edbbf5498..55f0fb67b2 100644 --- a/src/ProjectReferences.Persisters.Audit.props +++ b/src/ProjectReferences.Persisters.Audit.props @@ -3,7 +3,7 @@ - + \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/BodyStorage/PostgresqlAttachmentsBodyStorage.cs b/src/ServiceControl.Audit.Persistence.Postgresql/BodyStorage/PostgresqlAttachmentsBodyStorage.cs index 2ade14d8d7..25bea90a25 100644 --- a/src/ServiceControl.Audit.Persistence.Postgresql/BodyStorage/PostgresqlAttachmentsBodyStorage.cs +++ b/src/ServiceControl.Audit.Persistence.Postgresql/BodyStorage/PostgresqlAttachmentsBodyStorage.cs @@ -1,11 +1,11 @@ -namespace ServiceControl.Audit.Persistence.Postgresql.BodyStorage +namespace ServiceControl.Audit.Persistence.PostgreSQL.BodyStorage { using System.IO; using System.Threading; using System.Threading.Tasks; using ServiceControl.Audit.Auditing.BodyStorage; - public class PostgresqlAttachmentsBodyStorage : IBodyStorage + public class PostgreSQLAttachmentsBodyStorage : IBodyStorage { public Task Store(string bodyId, string contentType, int bodySize, Stream bodyStream, CancellationToken cancellationToken) => throw new System.NotImplementedException(); public Task TryFetch(string bodyId, CancellationToken cancellationToken) => throw new System.NotImplementedException(); diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgreSQLConnectionFactory.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgreSQLConnectionFactory.cs new file mode 100644 index 0000000000..a919ca1ce4 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Postgresql/PostgreSQLConnectionFactory.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.Audit.Persistence.PostgreSQL +{ + using Npgsql; + using System.Threading.Tasks; + using System.Threading; + + public class PostgreSQLConnectionFactory + { + readonly string connectionString; + + public PostgreSQLConnectionFactory(string connectionString) + { + this.connectionString = connectionString; + } + + public NpgsqlConnection CreateConnection() + { + return new NpgsqlConnection(connectionString); + } + + public async Task OpenConnectionAsync(CancellationToken cancellationToken = default) + { + var conn = new NpgsqlConnection(connectionString); + await conn.OpenAsync(cancellationToken).ConfigureAwait(false); + return conn; + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlAuditDataStore.cs index fdf8e99d72..2694d30c52 100644 --- a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlAuditDataStore.cs @@ -1,4 +1,4 @@ -namespace ServiceControl.Audit.Persistence.Postgresql +namespace ServiceControl.Audit.Persistence.PostgreSQL { using System; using System.Collections.Generic; @@ -11,7 +11,7 @@ namespace ServiceControl.Audit.Persistence.Postgresql using ServiceControl.Audit.Persistence; using ServiceControl.SagaAudit; - public class PostgresqlAuditDataStore : IAuditDataStore + public class PostgreSQLAuditDataStore : IAuditDataStore { public Task GetMessageBody(string messageId, CancellationToken cancellationToken) => throw new NotImplementedException(); public Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlFailedAuditStorage.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlFailedAuditStorage.cs index 7e1b7402f5..912c4e16b9 100644 --- a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlFailedAuditStorage.cs +++ b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlFailedAuditStorage.cs @@ -1,4 +1,4 @@ -namespace ServiceControl.Audit.Persistence.Postgresql +namespace ServiceControl.Audit.Persistence.PostgreSQL { using System; using System.Threading; @@ -6,7 +6,7 @@ namespace ServiceControl.Audit.Persistence.Postgresql using ServiceControl.Audit.Auditing; using ServiceControl.Audit.Persistence; - public class PostgresqlFailedAuditStorage : IFailedAuditStorage + public class PostgreSQLFailedAuditStorage : IFailedAuditStorage { public Task GetFailedAuditsCount() => throw new NotImplementedException(); public Task ProcessFailedMessages(Func, CancellationToken, Task> onMessage, CancellationToken cancellationToken) => throw new NotImplementedException(); diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistence.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistence.cs index 4325a528c6..fd3417dcd2 100644 --- a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistence.cs @@ -1,11 +1,26 @@ -namespace ServiceControl.Audit.Persistence.Postgresql +namespace ServiceControl.Audit.Persistence.PostgreSQL { using Microsoft.Extensions.DependencyInjection; + using ServiceControl.Audit.Auditing.BodyStorage; using ServiceControl.Audit.Persistence; + using ServiceControl.Audit.Persistence.PostgreSQL.BodyStorage; + using ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; + using ServiceControl.Audit.Persistence.UnitOfWork; - public class PostgresqlPersistence : IPersistence + public class PostgreSQLPersistence : IPersistence { - public void AddInstaller(IServiceCollection services) => throw new System.NotImplementedException(); - public void AddPersistence(IServiceCollection services) => throw new System.NotImplementedException(); + public void AddInstaller(IServiceCollection services) + { + AddPersistence(services); + } + + public void AddPersistence(IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + } } } diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistenceConfiguration.cs index 12091cc160..5c6319a495 100644 --- a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistenceConfiguration.cs +++ b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistenceConfiguration.cs @@ -1,14 +1,82 @@ -namespace ServiceControl.Audit.Persistence.Postgresql +namespace ServiceControl.Audit.Persistence.PostgreSQL { using System.Collections.Generic; + using Npgsql; using ServiceControl.Audit.Persistence; - public class PostgresqlPersistenceConfiguration : IPersistenceConfiguration + public class PostgreSQLPersistenceConfiguration : IPersistenceConfiguration { public string Name => "PostgreSQL"; - public IEnumerable ConfigurationKeys => new[] { "ConnectionString", "MaxConnections" }; + public IEnumerable ConfigurationKeys => new[] { "PostgreSqlConnectionString" }; - public IPersistence Create(PersistenceSettings settings) => throw new System.NotImplementedException(); + public IPersistence Create(PersistenceSettings settings) + { + settings.PersisterSpecificSettings.TryGetValue("PostgreSqlConnectionString", out var connectionString); + using var connection = new NpgsqlConnection(connectionString); + connection.Open(); + + // Create processed_messages table + using (var cmd = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS processed_messages ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + unique_message_id TEXT, + message_metadata JSONB, + headers JSONB, + processed_at TIMESTAMPTZ, + body BYTEA, + message_id TEXT, + message_type TEXT, + is_system_message BOOLEAN, + status TEXT, + time_sent TIMESTAMPTZ, + receiving_endpoint_name TEXT, + critical_time INTERVAL, + processing_time INTERVAL, + delivery_time INTERVAL, + conversation_id TEXT, + query tsvector GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(headers::text, '')), 'A') || + setweight(to_tsvector('english', coalesce(body::text, '')), 'B') + ) STORED + );", connection)) + { + cmd.ExecuteNonQuery(); + } + + // Create saga_snapshots table + using (var cmd = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS saga_snapshots ( + id TEXT PRIMARY KEY, + saga_id UUID, + saga_type TEXT, + start_time TIMESTAMPTZ, + finish_time TIMESTAMPTZ, + status TEXT, + state_after_change TEXT, + initiating_message JSONB, + outgoing_messages JSONB, + endpoint TEXT, + processed_at TIMESTAMPTZ + );", connection)) + { + cmd.ExecuteNonQuery(); + } + + // Create known_endpoints table + using (var cmd = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS known_endpoints ( + id TEXT PRIMARY KEY, + name TEXT, + host_id UUID, + host TEXT, + last_seen TIMESTAMPTZ + );", connection)) + { + cmd.ExecuteNonQuery(); + } + + return new PostgreSQLPersistence(); + } } } diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/ServiceControl.Audit.Persistence.Postgresql.csproj b/src/ServiceControl.Audit.Persistence.Postgresql/ServiceControl.Audit.Persistence.Postgresql.csproj index b2232a9098..ab9cfb0e27 100644 --- a/src/ServiceControl.Audit.Persistence.Postgresql/ServiceControl.Audit.Persistence.Postgresql.csproj +++ b/src/ServiceControl.Audit.Persistence.Postgresql/ServiceControl.Audit.Persistence.Postgresql.csproj @@ -8,7 +8,6 @@ - @@ -16,4 +15,14 @@ + + + + + + + + + + \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWork.cs index d1fd17f3eb..cdeea30e96 100644 --- a/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWork.cs @@ -1,29 +1,124 @@ -namespace ServiceControl.Audit.Persistence.Postgresql.UnitOfWork +namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork { using System; + using System.Text.Json; using System.Threading; using System.Threading.Tasks; + using Npgsql; using ServiceControl.Audit.Auditing; using ServiceControl.Audit.Persistence.Monitoring; using ServiceControl.Audit.Persistence.UnitOfWork; using ServiceControl.SagaAudit; - public class PostgresqlAuditIngestionUnitOfWork : IAuditIngestionUnitOfWork + public class PostgreSQLAuditIngestionUnitOfWork : IAuditIngestionUnitOfWork { - public ValueTask DisposeAsync() + readonly NpgsqlConnection connection; + readonly NpgsqlTransaction transaction; + + public PostgreSQLAuditIngestionUnitOfWork(NpgsqlConnection connection, NpgsqlTransaction transaction) + { + this.connection = connection; + this.transaction = transaction; + } + + public async ValueTask DisposeAsync() + { + await transaction.DisposeAsync().ConfigureAwait(false); + await connection.DisposeAsync().ConfigureAwait(false); + } + + public async Task CompleteAsync(CancellationToken cancellationToken = default) { - // TODO: Dispose resources if needed - return ValueTask.CompletedTask; + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); } - public Task CompleteAsync(CancellationToken cancellationToken = default) + public async Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body, CancellationToken cancellationToken = default) { - // TODO: Commit transaction or batch - throw new NotImplementedException(); + object GetMetadata(string key) => processedMessage.MessageMetadata.TryGetValue(key, out var value) ? value ?? DBNull.Value : DBNull.Value; + + // Insert ProcessedMessage into processed_messages table + var cmd = new NpgsqlCommand(@" + INSERT INTO processed_messages ( + unique_message_id, message_metadata, headers, processed_at, body, + message_id, message_type, is_system_message, status, time_sent, receiving_endpoint_name, + critical_time, processing_time, delivery_time, conversation_id + ) VALUES ( + @unique_message_id, @message_metadata, @headers, @processed_at, @body, + @message_id, @message_type, @is_system_message, @status, @time_sent, @receiving_endpoint_name, + @critical_time, @processing_time, @delivery_time, @conversation_id + ) + ;", connection, transaction); + + processedMessage.MessageMetadata["ContentLength"] = body.Length; + if (!body.IsEmpty) + { + cmd.Parameters.AddWithValue("body", body); + } + else + { + cmd.Parameters.AddWithValue("body", DBNull.Value); + } + cmd.Parameters.AddWithValue("unique_message_id", processedMessage.UniqueMessageId ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("message_metadata", JsonSerializer.Serialize(processedMessage.MessageMetadata)); + cmd.Parameters.AddWithValue("headers", JsonSerializer.Serialize(processedMessage.Headers)); + cmd.Parameters.AddWithValue("processed_at", processedMessage.ProcessedAt); + cmd.Parameters.AddWithValue("message_id", GetMetadata("MessageId")); + cmd.Parameters.AddWithValue("message_type", GetMetadata("MessageType")); + cmd.Parameters.AddWithValue("is_system_message", GetMetadata("IsSystemMessage")); + cmd.Parameters.AddWithValue("time_sent", GetMetadata("TimeSent")); + cmd.Parameters.AddWithValue("receiving_endpoint_name", GetMetadata("ReceivingEndpoint")); + cmd.Parameters.AddWithValue("critical_time", GetMetadata("CriticalTime")); + cmd.Parameters.AddWithValue("processing_time", GetMetadata("ProcessingTime")); + cmd.Parameters.AddWithValue("delivery_time", GetMetadata("DeliveryTime")); + cmd.Parameters.AddWithValue("conversation_id", GetMetadata("ConversationId")); + + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } - public Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body = default, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public async Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken = default) + { + // Insert SagaSnapshot into saga_snapshots table + var cmd = new NpgsqlCommand(@" + INSERT INTO saga_snapshots ( + id, saga_id, saga_type, start_time, finish_time, status, state_after_change, initiating_message, outgoing_messages, endpoint, processed_at + ) VALUES ( + @id, @saga_id, @saga_type, @start_time, @finish_time, @status, @state_after_change, @initiating_message, @outgoing_messages, @endpoint, @processed_at + ) + ON CONFLICT (id) DO NOTHING;", connection, transaction); + + cmd.Parameters.AddWithValue("id", sagaSnapshot.Id ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("saga_id", sagaSnapshot.SagaId); + cmd.Parameters.AddWithValue("saga_type", sagaSnapshot.SagaType ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("start_time", sagaSnapshot.StartTime); + cmd.Parameters.AddWithValue("finish_time", sagaSnapshot.FinishTime); + cmd.Parameters.AddWithValue("status", sagaSnapshot.Status.ToString()); + cmd.Parameters.AddWithValue("state_after_change", sagaSnapshot.StateAfterChange ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("initiating_message", JsonSerializer.Serialize(sagaSnapshot.InitiatingMessage)); + cmd.Parameters.AddWithValue("outgoing_messages", JsonSerializer.Serialize(sagaSnapshot.OutgoingMessages)); + cmd.Parameters.AddWithValue("endpoint", sagaSnapshot.Endpoint ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("processed_at", sagaSnapshot.ProcessedAt); + + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + public async Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken = default) + { + // Insert KnownEndpoint into known_endpoints table + var cmd = new NpgsqlCommand(@" + INSERT INTO known_endpoints ( + id, name, host_id, host, last_seen + ) VALUES ( + @id, @name, @host_id, @host, @last_seen + ) + ON CONFLICT (id) DO NOTHING;", connection, transaction); + + cmd.Parameters.AddWithValue("id", knownEndpoint.Id ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("name", knownEndpoint.Name ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("host_id", knownEndpoint.HostId); + cmd.Parameters.AddWithValue("host", knownEndpoint.Host ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("last_seen", knownEndpoint.LastSeen); + + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } } } diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWorkFactory.cs index e80fcee552..7b76ed259f 100644 --- a/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWorkFactory.cs +++ b/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWorkFactory.cs @@ -1,15 +1,25 @@ -namespace ServiceControl.Audit.Persistence.Postgresql.UnitOfWork +namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork { using System.Threading; using System.Threading.Tasks; + using Npgsql; using ServiceControl.Audit.Persistence.UnitOfWork; + using ServiceControl.Audit.Persistence.PostgreSQL; - public class PostgresqlAuditIngestionUnitOfWorkFactory : IAuditIngestionUnitOfWorkFactory + public class PostgreSQLAuditIngestionUnitOfWorkFactory : IAuditIngestionUnitOfWorkFactory { - public ValueTask StartNew(int batchSize, CancellationToken cancellationToken) + readonly PostgreSQLConnectionFactory connectionFactory; + + public PostgreSQLAuditIngestionUnitOfWorkFactory(PostgreSQLConnectionFactory connectionFactory) + { + this.connectionFactory = connectionFactory; + } + + public async ValueTask StartNew(int batchSize, CancellationToken cancellationToken) { - // TODO: Implement logic to start a new unit of work for PostgreSQL - throw new System.NotImplementedException(); + var connection = await connectionFactory.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + return new PostgreSQLAuditIngestionUnitOfWork(connection, transaction); } public bool CanIngestMore() => true; // TODO: Implement logic based on storage state diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/persistence.manifest b/src/ServiceControl.Audit.Persistence.Postgresql/persistence.manifest index b1b970370c..8c29a40f00 100644 --- a/src/ServiceControl.Audit.Persistence.Postgresql/persistence.manifest +++ b/src/ServiceControl.Audit.Persistence.Postgresql/persistence.manifest @@ -1,7 +1,7 @@ { - "Name": "Postgresql", - "DisplayName": "Postgresql", - "Description": "Postgresql ServiceControl Audit persister", - "AssemblyName": "ServiceControl.Audit.Persistence.Postgresql", - "TypeName": "ServiceControl.Audit.Persistence.Postgresql.PostgresqlPersistenceConfiguration, ServiceControl.Audit.Persistence.Postgresql" + "Name": "PostgreSQL", + "DisplayName": "PostgreSQL", + "Description": "PostgreSQL ServiceControl Audit persister", + "AssemblyName": "ServiceControl.Audit.Persistence.PostgreSQL", + "TypeName": "ServiceControl.Audit.Persistence.PostgreSQL.PostgreSQLPersistenceConfiguration, ServiceControl.Audit.Persistence.PostgreSQL" } \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence/DevelopmentPersistenceLocations.cs b/src/ServiceControl.Audit.Persistence/DevelopmentPersistenceLocations.cs index 08925bde80..4853ae65f3 100644 --- a/src/ServiceControl.Audit.Persistence/DevelopmentPersistenceLocations.cs +++ b/src/ServiceControl.Audit.Persistence/DevelopmentPersistenceLocations.cs @@ -19,6 +19,7 @@ static DevelopmentPersistenceLocations() { ManifestFiles.Add(BuildManifestPath(srcFolder, "ServiceControl.Audit.Persistence.InMemory")); ManifestFiles.Add(BuildManifestPath(srcFolder, "ServiceControl.Audit.Persistence.RavenDB")); + ManifestFiles.Add(BuildManifestPath(srcFolder, "ServiceControl.Audit.Persistence.PostgreSQL")); } } diff --git a/src/ServiceControl.sln b/src/ServiceControl.sln index 19e85db8c0..3f554852d7 100644 --- a/src/ServiceControl.sln +++ b/src/ServiceControl.sln @@ -188,7 +188,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceControl.Hosting", "S EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SetupProcessFake", "SetupProcessFake\SetupProcessFake.csproj", "{5837F789-69B9-44BE-B114-3A2880F06CAB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceControl.Audit.Persistence.Postgresql", "ServiceControl.Audit.Persistence.Postgresql\ServiceControl.Audit.Persistence.Postgresql.csproj", "{AD3CA060-83E5-4E39-82A3-774023B057DF}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceControl.Audit.Persistence.PostgresSQL", "ServiceControl.Audit.Persistence.PostgreSQL\ServiceControl.Audit.Persistence.PostgreSQL.csproj", "{AD3CA060-83E5-4E39-82A3-774023B057DF}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution From 5f091b1486a89f469c0a84599855da83314f5f8c Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 27 Aug 2025 09:21:22 +1000 Subject: [PATCH 03/41] Remove to readd --- .../PostgresqlAttachmentsBodyStorage.cs | 13 -- .../PostgreSQLConnectionFactory.cs | 28 ---- .../PostgresqlAuditDataStore.cs | 26 ---- .../PostgresqlFailedAuditStorage.cs | 15 --- .../PostgresqlPersistence.cs | 26 ---- .../PostgresqlPersistenceConfiguration.cs | 82 ------------ ...ontrol.Audit.Persistence.Postgresql.csproj | 28 ---- .../PostgresqlAuditIngestionUnitOfWork.cs | 124 ------------------ ...stgresqlAuditIngestionUnitOfWorkFactory.cs | 27 ---- .../persistence.manifest | 7 - src/ServiceControl.sln | 15 --- 11 files changed, 391 deletions(-) delete mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/BodyStorage/PostgresqlAttachmentsBodyStorage.cs delete mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/PostgreSQLConnectionFactory.cs delete mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlAuditDataStore.cs delete mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlFailedAuditStorage.cs delete mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistence.cs delete mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistenceConfiguration.cs delete mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/ServiceControl.Audit.Persistence.Postgresql.csproj delete mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWork.cs delete mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWorkFactory.cs delete mode 100644 src/ServiceControl.Audit.Persistence.Postgresql/persistence.manifest diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/BodyStorage/PostgresqlAttachmentsBodyStorage.cs b/src/ServiceControl.Audit.Persistence.Postgresql/BodyStorage/PostgresqlAttachmentsBodyStorage.cs deleted file mode 100644 index 25bea90a25..0000000000 --- a/src/ServiceControl.Audit.Persistence.Postgresql/BodyStorage/PostgresqlAttachmentsBodyStorage.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL.BodyStorage -{ - using System.IO; - using System.Threading; - using System.Threading.Tasks; - using ServiceControl.Audit.Auditing.BodyStorage; - - public class PostgreSQLAttachmentsBodyStorage : IBodyStorage - { - public Task Store(string bodyId, string contentType, int bodySize, Stream bodyStream, CancellationToken cancellationToken) => throw new System.NotImplementedException(); - public Task TryFetch(string bodyId, CancellationToken cancellationToken) => throw new System.NotImplementedException(); - } -} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgreSQLConnectionFactory.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgreSQLConnectionFactory.cs deleted file mode 100644 index a919ca1ce4..0000000000 --- a/src/ServiceControl.Audit.Persistence.Postgresql/PostgreSQLConnectionFactory.cs +++ /dev/null @@ -1,28 +0,0 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL -{ - using Npgsql; - using System.Threading.Tasks; - using System.Threading; - - public class PostgreSQLConnectionFactory - { - readonly string connectionString; - - public PostgreSQLConnectionFactory(string connectionString) - { - this.connectionString = connectionString; - } - - public NpgsqlConnection CreateConnection() - { - return new NpgsqlConnection(connectionString); - } - - public async Task OpenConnectionAsync(CancellationToken cancellationToken = default) - { - var conn = new NpgsqlConnection(connectionString); - await conn.OpenAsync(cancellationToken).ConfigureAwait(false); - return conn; - } - } -} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlAuditDataStore.cs deleted file mode 100644 index 2694d30c52..0000000000 --- a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlAuditDataStore.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL -{ - using System; - using System.Collections.Generic; - using System.Threading; - using System.Threading.Tasks; - using ServiceControl.Audit.Auditing; - using ServiceControl.Audit.Auditing.MessagesView; - using ServiceControl.Audit.Infrastructure; - using ServiceControl.Audit.Monitoring; - using ServiceControl.Audit.Persistence; - using ServiceControl.SagaAudit; - - public class PostgreSQLAuditDataStore : IAuditDataStore - { - public Task GetMessageBody(string messageId, CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task>> QueryKnownEndpoints(CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task> QuerySagaHistoryById(Guid input, CancellationToken cancellationToken) => throw new NotImplementedException(); - } -} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlFailedAuditStorage.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlFailedAuditStorage.cs deleted file mode 100644 index 912c4e16b9..0000000000 --- a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlFailedAuditStorage.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL -{ - using System; - using System.Threading; - using System.Threading.Tasks; - using ServiceControl.Audit.Auditing; - using ServiceControl.Audit.Persistence; - - public class PostgreSQLFailedAuditStorage : IFailedAuditStorage - { - public Task GetFailedAuditsCount() => throw new NotImplementedException(); - public Task ProcessFailedMessages(Func, CancellationToken, Task> onMessage, CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task SaveFailedAuditImport(FailedAuditImport message) => throw new NotImplementedException(); - } -} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistence.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistence.cs deleted file mode 100644 index fd3417dcd2..0000000000 --- a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistence.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL -{ - using Microsoft.Extensions.DependencyInjection; - using ServiceControl.Audit.Auditing.BodyStorage; - using ServiceControl.Audit.Persistence; - using ServiceControl.Audit.Persistence.PostgreSQL.BodyStorage; - using ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; - using ServiceControl.Audit.Persistence.UnitOfWork; - - public class PostgreSQLPersistence : IPersistence - { - public void AddInstaller(IServiceCollection services) - { - AddPersistence(services); - } - - public void AddPersistence(IServiceCollection services) - { - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - } - } -} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistenceConfiguration.cs deleted file mode 100644 index 5c6319a495..0000000000 --- a/src/ServiceControl.Audit.Persistence.Postgresql/PostgresqlPersistenceConfiguration.cs +++ /dev/null @@ -1,82 +0,0 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL -{ - using System.Collections.Generic; - using Npgsql; - using ServiceControl.Audit.Persistence; - - public class PostgreSQLPersistenceConfiguration : IPersistenceConfiguration - { - public string Name => "PostgreSQL"; - - public IEnumerable ConfigurationKeys => new[] { "PostgreSqlConnectionString" }; - - public IPersistence Create(PersistenceSettings settings) - { - settings.PersisterSpecificSettings.TryGetValue("PostgreSqlConnectionString", out var connectionString); - using var connection = new NpgsqlConnection(connectionString); - connection.Open(); - - // Create processed_messages table - using (var cmd = new NpgsqlCommand(@" - CREATE TABLE IF NOT EXISTS processed_messages ( - id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - unique_message_id TEXT, - message_metadata JSONB, - headers JSONB, - processed_at TIMESTAMPTZ, - body BYTEA, - message_id TEXT, - message_type TEXT, - is_system_message BOOLEAN, - status TEXT, - time_sent TIMESTAMPTZ, - receiving_endpoint_name TEXT, - critical_time INTERVAL, - processing_time INTERVAL, - delivery_time INTERVAL, - conversation_id TEXT, - query tsvector GENERATED ALWAYS AS ( - setweight(to_tsvector('english', coalesce(headers::text, '')), 'A') || - setweight(to_tsvector('english', coalesce(body::text, '')), 'B') - ) STORED - );", connection)) - { - cmd.ExecuteNonQuery(); - } - - // Create saga_snapshots table - using (var cmd = new NpgsqlCommand(@" - CREATE TABLE IF NOT EXISTS saga_snapshots ( - id TEXT PRIMARY KEY, - saga_id UUID, - saga_type TEXT, - start_time TIMESTAMPTZ, - finish_time TIMESTAMPTZ, - status TEXT, - state_after_change TEXT, - initiating_message JSONB, - outgoing_messages JSONB, - endpoint TEXT, - processed_at TIMESTAMPTZ - );", connection)) - { - cmd.ExecuteNonQuery(); - } - - // Create known_endpoints table - using (var cmd = new NpgsqlCommand(@" - CREATE TABLE IF NOT EXISTS known_endpoints ( - id TEXT PRIMARY KEY, - name TEXT, - host_id UUID, - host TEXT, - last_seen TIMESTAMPTZ - );", connection)) - { - cmd.ExecuteNonQuery(); - } - - return new PostgreSQLPersistence(); - } - } -} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/ServiceControl.Audit.Persistence.Postgresql.csproj b/src/ServiceControl.Audit.Persistence.Postgresql/ServiceControl.Audit.Persistence.Postgresql.csproj deleted file mode 100644 index ab9cfb0e27..0000000000 --- a/src/ServiceControl.Audit.Persistence.Postgresql/ServiceControl.Audit.Persistence.Postgresql.csproj +++ /dev/null @@ -1,28 +0,0 @@ - - - - net8.0 - true - true - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWork.cs deleted file mode 100644 index cdeea30e96..0000000000 --- a/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWork.cs +++ /dev/null @@ -1,124 +0,0 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork -{ - using System; - using System.Text.Json; - using System.Threading; - using System.Threading.Tasks; - using Npgsql; - using ServiceControl.Audit.Auditing; - using ServiceControl.Audit.Persistence.Monitoring; - using ServiceControl.Audit.Persistence.UnitOfWork; - using ServiceControl.SagaAudit; - - public class PostgreSQLAuditIngestionUnitOfWork : IAuditIngestionUnitOfWork - { - readonly NpgsqlConnection connection; - readonly NpgsqlTransaction transaction; - - public PostgreSQLAuditIngestionUnitOfWork(NpgsqlConnection connection, NpgsqlTransaction transaction) - { - this.connection = connection; - this.transaction = transaction; - } - - public async ValueTask DisposeAsync() - { - await transaction.DisposeAsync().ConfigureAwait(false); - await connection.DisposeAsync().ConfigureAwait(false); - } - - public async Task CompleteAsync(CancellationToken cancellationToken = default) - { - await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); - } - - public async Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body, CancellationToken cancellationToken = default) - { - object GetMetadata(string key) => processedMessage.MessageMetadata.TryGetValue(key, out var value) ? value ?? DBNull.Value : DBNull.Value; - - // Insert ProcessedMessage into processed_messages table - var cmd = new NpgsqlCommand(@" - INSERT INTO processed_messages ( - unique_message_id, message_metadata, headers, processed_at, body, - message_id, message_type, is_system_message, status, time_sent, receiving_endpoint_name, - critical_time, processing_time, delivery_time, conversation_id - ) VALUES ( - @unique_message_id, @message_metadata, @headers, @processed_at, @body, - @message_id, @message_type, @is_system_message, @status, @time_sent, @receiving_endpoint_name, - @critical_time, @processing_time, @delivery_time, @conversation_id - ) - ;", connection, transaction); - - processedMessage.MessageMetadata["ContentLength"] = body.Length; - if (!body.IsEmpty) - { - cmd.Parameters.AddWithValue("body", body); - } - else - { - cmd.Parameters.AddWithValue("body", DBNull.Value); - } - cmd.Parameters.AddWithValue("unique_message_id", processedMessage.UniqueMessageId ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("message_metadata", JsonSerializer.Serialize(processedMessage.MessageMetadata)); - cmd.Parameters.AddWithValue("headers", JsonSerializer.Serialize(processedMessage.Headers)); - cmd.Parameters.AddWithValue("processed_at", processedMessage.ProcessedAt); - cmd.Parameters.AddWithValue("message_id", GetMetadata("MessageId")); - cmd.Parameters.AddWithValue("message_type", GetMetadata("MessageType")); - cmd.Parameters.AddWithValue("is_system_message", GetMetadata("IsSystemMessage")); - cmd.Parameters.AddWithValue("time_sent", GetMetadata("TimeSent")); - cmd.Parameters.AddWithValue("receiving_endpoint_name", GetMetadata("ReceivingEndpoint")); - cmd.Parameters.AddWithValue("critical_time", GetMetadata("CriticalTime")); - cmd.Parameters.AddWithValue("processing_time", GetMetadata("ProcessingTime")); - cmd.Parameters.AddWithValue("delivery_time", GetMetadata("DeliveryTime")); - cmd.Parameters.AddWithValue("conversation_id", GetMetadata("ConversationId")); - - await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); - } - - public async Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken = default) - { - // Insert SagaSnapshot into saga_snapshots table - var cmd = new NpgsqlCommand(@" - INSERT INTO saga_snapshots ( - id, saga_id, saga_type, start_time, finish_time, status, state_after_change, initiating_message, outgoing_messages, endpoint, processed_at - ) VALUES ( - @id, @saga_id, @saga_type, @start_time, @finish_time, @status, @state_after_change, @initiating_message, @outgoing_messages, @endpoint, @processed_at - ) - ON CONFLICT (id) DO NOTHING;", connection, transaction); - - cmd.Parameters.AddWithValue("id", sagaSnapshot.Id ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("saga_id", sagaSnapshot.SagaId); - cmd.Parameters.AddWithValue("saga_type", sagaSnapshot.SagaType ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("start_time", sagaSnapshot.StartTime); - cmd.Parameters.AddWithValue("finish_time", sagaSnapshot.FinishTime); - cmd.Parameters.AddWithValue("status", sagaSnapshot.Status.ToString()); - cmd.Parameters.AddWithValue("state_after_change", sagaSnapshot.StateAfterChange ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("initiating_message", JsonSerializer.Serialize(sagaSnapshot.InitiatingMessage)); - cmd.Parameters.AddWithValue("outgoing_messages", JsonSerializer.Serialize(sagaSnapshot.OutgoingMessages)); - cmd.Parameters.AddWithValue("endpoint", sagaSnapshot.Endpoint ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("processed_at", sagaSnapshot.ProcessedAt); - - await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); - } - - public async Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken = default) - { - // Insert KnownEndpoint into known_endpoints table - var cmd = new NpgsqlCommand(@" - INSERT INTO known_endpoints ( - id, name, host_id, host, last_seen - ) VALUES ( - @id, @name, @host_id, @host, @last_seen - ) - ON CONFLICT (id) DO NOTHING;", connection, transaction); - - cmd.Parameters.AddWithValue("id", knownEndpoint.Id ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("name", knownEndpoint.Name ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("host_id", knownEndpoint.HostId); - cmd.Parameters.AddWithValue("host", knownEndpoint.Host ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("last_seen", knownEndpoint.LastSeen); - - await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); - } - } -} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWorkFactory.cs deleted file mode 100644 index 7b76ed259f..0000000000 --- a/src/ServiceControl.Audit.Persistence.Postgresql/UnitOfWork/PostgresqlAuditIngestionUnitOfWorkFactory.cs +++ /dev/null @@ -1,27 +0,0 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork -{ - using System.Threading; - using System.Threading.Tasks; - using Npgsql; - using ServiceControl.Audit.Persistence.UnitOfWork; - using ServiceControl.Audit.Persistence.PostgreSQL; - - public class PostgreSQLAuditIngestionUnitOfWorkFactory : IAuditIngestionUnitOfWorkFactory - { - readonly PostgreSQLConnectionFactory connectionFactory; - - public PostgreSQLAuditIngestionUnitOfWorkFactory(PostgreSQLConnectionFactory connectionFactory) - { - this.connectionFactory = connectionFactory; - } - - public async ValueTask StartNew(int batchSize, CancellationToken cancellationToken) - { - var connection = await connectionFactory.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); - var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); - return new PostgreSQLAuditIngestionUnitOfWork(connection, transaction); - } - - public bool CanIngestMore() => true; // TODO: Implement logic based on storage state - } -} diff --git a/src/ServiceControl.Audit.Persistence.Postgresql/persistence.manifest b/src/ServiceControl.Audit.Persistence.Postgresql/persistence.manifest deleted file mode 100644 index 8c29a40f00..0000000000 --- a/src/ServiceControl.Audit.Persistence.Postgresql/persistence.manifest +++ /dev/null @@ -1,7 +0,0 @@ -{ - "Name": "PostgreSQL", - "DisplayName": "PostgreSQL", - "Description": "PostgreSQL ServiceControl Audit persister", - "AssemblyName": "ServiceControl.Audit.Persistence.PostgreSQL", - "TypeName": "ServiceControl.Audit.Persistence.PostgreSQL.PostgreSQLPersistenceConfiguration, ServiceControl.Audit.Persistence.PostgreSQL" -} \ No newline at end of file diff --git a/src/ServiceControl.sln b/src/ServiceControl.sln index 3f554852d7..5a068daa4d 100644 --- a/src/ServiceControl.sln +++ b/src/ServiceControl.sln @@ -188,8 +188,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceControl.Hosting", "S EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SetupProcessFake", "SetupProcessFake\SetupProcessFake.csproj", "{5837F789-69B9-44BE-B114-3A2880F06CAB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceControl.Audit.Persistence.PostgresSQL", "ServiceControl.Audit.Persistence.PostgreSQL\ServiceControl.Audit.Persistence.PostgreSQL.csproj", "{AD3CA060-83E5-4E39-82A3-774023B057DF}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1028,18 +1026,6 @@ Global {5837F789-69B9-44BE-B114-3A2880F06CAB}.Release|x64.Build.0 = Release|Any CPU {5837F789-69B9-44BE-B114-3A2880F06CAB}.Release|x86.ActiveCfg = Release|Any CPU {5837F789-69B9-44BE-B114-3A2880F06CAB}.Release|x86.Build.0 = Release|Any CPU - {AD3CA060-83E5-4E39-82A3-774023B057DF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {AD3CA060-83E5-4E39-82A3-774023B057DF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {AD3CA060-83E5-4E39-82A3-774023B057DF}.Debug|x64.ActiveCfg = Debug|Any CPU - {AD3CA060-83E5-4E39-82A3-774023B057DF}.Debug|x64.Build.0 = Debug|Any CPU - {AD3CA060-83E5-4E39-82A3-774023B057DF}.Debug|x86.ActiveCfg = Debug|Any CPU - {AD3CA060-83E5-4E39-82A3-774023B057DF}.Debug|x86.Build.0 = Debug|Any CPU - {AD3CA060-83E5-4E39-82A3-774023B057DF}.Release|Any CPU.ActiveCfg = Release|Any CPU - {AD3CA060-83E5-4E39-82A3-774023B057DF}.Release|Any CPU.Build.0 = Release|Any CPU - {AD3CA060-83E5-4E39-82A3-774023B057DF}.Release|x64.ActiveCfg = Release|Any CPU - {AD3CA060-83E5-4E39-82A3-774023B057DF}.Release|x64.Build.0 = Release|Any CPU - {AD3CA060-83E5-4E39-82A3-774023B057DF}.Release|x86.ActiveCfg = Release|Any CPU - {AD3CA060-83E5-4E39-82A3-774023B057DF}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1125,7 +1111,6 @@ Global {18DBEEF5-42EE-4C1D-A05B-87B21C067D53} = {E0E45F22-35E3-4AD8-B09E-EFEA5A2F18EE} {481032A1-1106-4C6C-B75E-512F2FB08882} = {9AF9D3C7-E859-451B-BA4D-B954D289213A} {5837F789-69B9-44BE-B114-3A2880F06CAB} = {927A078A-E271-4878-A153-86D71AE510E2} - {AD3CA060-83E5-4E39-82A3-774023B057DF} = {BD162BC6-705F-45B4-A6B5-C138DC966C1D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3B9E5B72-F580-465A-A22C-2D2148AF4EB4} From 684d265a8313f496e7dd521cb529a818b1221049 Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 27 Aug 2025 09:26:12 +1000 Subject: [PATCH 04/41] Adding it back --- .../PostgreSQLAttachmentsBodyStorage.cs | 13 ++ .../PostgreSQLAuditDataStore.cs | 26 ++++ .../PostgreSQLConnectionFactory.cs | 28 ++++ .../PostgreSQLFailedAuditStorage.cs | 15 +++ .../PostgreSQLPersistence.cs | 26 ++++ .../PostgreSQLPersistenceConfiguration.cs | 82 ++++++++++++ ...ontrol.Audit.Persistence.PostgreSQL.csproj | 28 ++++ .../PostgreSQLAuditIngestionUnitOfWork.cs | 124 ++++++++++++++++++ ...stgreSQLAuditIngestionUnitOfWorkFactory.cs | 27 ++++ .../persistence.manifest | 7 + src/ServiceControl.sln | 15 +++ 11 files changed, 391 insertions(+) create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/BodyStorage/PostgreSQLAttachmentsBodyStorage.cs create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceConfiguration.cs create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/persistence.manifest diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/BodyStorage/PostgreSQLAttachmentsBodyStorage.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/BodyStorage/PostgreSQLAttachmentsBodyStorage.cs new file mode 100644 index 0000000000..25bea90a25 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/BodyStorage/PostgreSQLAttachmentsBodyStorage.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Audit.Persistence.PostgreSQL.BodyStorage +{ + using System.IO; + using System.Threading; + using System.Threading.Tasks; + using ServiceControl.Audit.Auditing.BodyStorage; + + public class PostgreSQLAttachmentsBodyStorage : IBodyStorage + { + public Task Store(string bodyId, string contentType, int bodySize, Stream bodyStream, CancellationToken cancellationToken) => throw new System.NotImplementedException(); + public Task TryFetch(string bodyId, CancellationToken cancellationToken) => throw new System.NotImplementedException(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs new file mode 100644 index 0000000000..2694d30c52 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Audit.Persistence.PostgreSQL +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using ServiceControl.Audit.Auditing; + using ServiceControl.Audit.Auditing.MessagesView; + using ServiceControl.Audit.Infrastructure; + using ServiceControl.Audit.Monitoring; + using ServiceControl.Audit.Persistence; + using ServiceControl.SagaAudit; + + public class PostgreSQLAuditDataStore : IAuditDataStore + { + public Task GetMessageBody(string messageId, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task>> QueryKnownEndpoints(CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task> QuerySagaHistoryById(Guid input, CancellationToken cancellationToken) => throw new NotImplementedException(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs new file mode 100644 index 0000000000..a919ca1ce4 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.Audit.Persistence.PostgreSQL +{ + using Npgsql; + using System.Threading.Tasks; + using System.Threading; + + public class PostgreSQLConnectionFactory + { + readonly string connectionString; + + public PostgreSQLConnectionFactory(string connectionString) + { + this.connectionString = connectionString; + } + + public NpgsqlConnection CreateConnection() + { + return new NpgsqlConnection(connectionString); + } + + public async Task OpenConnectionAsync(CancellationToken cancellationToken = default) + { + var conn = new NpgsqlConnection(connectionString); + await conn.OpenAsync(cancellationToken).ConfigureAwait(false); + return conn; + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs new file mode 100644 index 0000000000..912c4e16b9 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Audit.Persistence.PostgreSQL +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using ServiceControl.Audit.Auditing; + using ServiceControl.Audit.Persistence; + + public class PostgreSQLFailedAuditStorage : IFailedAuditStorage + { + public Task GetFailedAuditsCount() => throw new NotImplementedException(); + public Task ProcessFailedMessages(Func, CancellationToken, Task> onMessage, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task SaveFailedAuditImport(FailedAuditImport message) => throw new NotImplementedException(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs new file mode 100644 index 0000000000..fd3417dcd2 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Audit.Persistence.PostgreSQL +{ + using Microsoft.Extensions.DependencyInjection; + using ServiceControl.Audit.Auditing.BodyStorage; + using ServiceControl.Audit.Persistence; + using ServiceControl.Audit.Persistence.PostgreSQL.BodyStorage; + using ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; + using ServiceControl.Audit.Persistence.UnitOfWork; + + public class PostgreSQLPersistence : IPersistence + { + public void AddInstaller(IServiceCollection services) + { + AddPersistence(services); + } + + public void AddPersistence(IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceConfiguration.cs new file mode 100644 index 0000000000..5c6319a495 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceConfiguration.cs @@ -0,0 +1,82 @@ +namespace ServiceControl.Audit.Persistence.PostgreSQL +{ + using System.Collections.Generic; + using Npgsql; + using ServiceControl.Audit.Persistence; + + public class PostgreSQLPersistenceConfiguration : IPersistenceConfiguration + { + public string Name => "PostgreSQL"; + + public IEnumerable ConfigurationKeys => new[] { "PostgreSqlConnectionString" }; + + public IPersistence Create(PersistenceSettings settings) + { + settings.PersisterSpecificSettings.TryGetValue("PostgreSqlConnectionString", out var connectionString); + using var connection = new NpgsqlConnection(connectionString); + connection.Open(); + + // Create processed_messages table + using (var cmd = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS processed_messages ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + unique_message_id TEXT, + message_metadata JSONB, + headers JSONB, + processed_at TIMESTAMPTZ, + body BYTEA, + message_id TEXT, + message_type TEXT, + is_system_message BOOLEAN, + status TEXT, + time_sent TIMESTAMPTZ, + receiving_endpoint_name TEXT, + critical_time INTERVAL, + processing_time INTERVAL, + delivery_time INTERVAL, + conversation_id TEXT, + query tsvector GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(headers::text, '')), 'A') || + setweight(to_tsvector('english', coalesce(body::text, '')), 'B') + ) STORED + );", connection)) + { + cmd.ExecuteNonQuery(); + } + + // Create saga_snapshots table + using (var cmd = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS saga_snapshots ( + id TEXT PRIMARY KEY, + saga_id UUID, + saga_type TEXT, + start_time TIMESTAMPTZ, + finish_time TIMESTAMPTZ, + status TEXT, + state_after_change TEXT, + initiating_message JSONB, + outgoing_messages JSONB, + endpoint TEXT, + processed_at TIMESTAMPTZ + );", connection)) + { + cmd.ExecuteNonQuery(); + } + + // Create known_endpoints table + using (var cmd = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS known_endpoints ( + id TEXT PRIMARY KEY, + name TEXT, + host_id UUID, + host TEXT, + last_seen TIMESTAMPTZ + );", connection)) + { + cmd.ExecuteNonQuery(); + } + + return new PostgreSQLPersistence(); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj b/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj new file mode 100644 index 0000000000..678bd2b03d --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj @@ -0,0 +1,28 @@ + + + + net8.0 + true + true + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs new file mode 100644 index 0000000000..cdeea30e96 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -0,0 +1,124 @@ +namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork +{ + using System; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + using Npgsql; + using ServiceControl.Audit.Auditing; + using ServiceControl.Audit.Persistence.Monitoring; + using ServiceControl.Audit.Persistence.UnitOfWork; + using ServiceControl.SagaAudit; + + public class PostgreSQLAuditIngestionUnitOfWork : IAuditIngestionUnitOfWork + { + readonly NpgsqlConnection connection; + readonly NpgsqlTransaction transaction; + + public PostgreSQLAuditIngestionUnitOfWork(NpgsqlConnection connection, NpgsqlTransaction transaction) + { + this.connection = connection; + this.transaction = transaction; + } + + public async ValueTask DisposeAsync() + { + await transaction.DisposeAsync().ConfigureAwait(false); + await connection.DisposeAsync().ConfigureAwait(false); + } + + public async Task CompleteAsync(CancellationToken cancellationToken = default) + { + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + + public async Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body, CancellationToken cancellationToken = default) + { + object GetMetadata(string key) => processedMessage.MessageMetadata.TryGetValue(key, out var value) ? value ?? DBNull.Value : DBNull.Value; + + // Insert ProcessedMessage into processed_messages table + var cmd = new NpgsqlCommand(@" + INSERT INTO processed_messages ( + unique_message_id, message_metadata, headers, processed_at, body, + message_id, message_type, is_system_message, status, time_sent, receiving_endpoint_name, + critical_time, processing_time, delivery_time, conversation_id + ) VALUES ( + @unique_message_id, @message_metadata, @headers, @processed_at, @body, + @message_id, @message_type, @is_system_message, @status, @time_sent, @receiving_endpoint_name, + @critical_time, @processing_time, @delivery_time, @conversation_id + ) + ;", connection, transaction); + + processedMessage.MessageMetadata["ContentLength"] = body.Length; + if (!body.IsEmpty) + { + cmd.Parameters.AddWithValue("body", body); + } + else + { + cmd.Parameters.AddWithValue("body", DBNull.Value); + } + cmd.Parameters.AddWithValue("unique_message_id", processedMessage.UniqueMessageId ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("message_metadata", JsonSerializer.Serialize(processedMessage.MessageMetadata)); + cmd.Parameters.AddWithValue("headers", JsonSerializer.Serialize(processedMessage.Headers)); + cmd.Parameters.AddWithValue("processed_at", processedMessage.ProcessedAt); + cmd.Parameters.AddWithValue("message_id", GetMetadata("MessageId")); + cmd.Parameters.AddWithValue("message_type", GetMetadata("MessageType")); + cmd.Parameters.AddWithValue("is_system_message", GetMetadata("IsSystemMessage")); + cmd.Parameters.AddWithValue("time_sent", GetMetadata("TimeSent")); + cmd.Parameters.AddWithValue("receiving_endpoint_name", GetMetadata("ReceivingEndpoint")); + cmd.Parameters.AddWithValue("critical_time", GetMetadata("CriticalTime")); + cmd.Parameters.AddWithValue("processing_time", GetMetadata("ProcessingTime")); + cmd.Parameters.AddWithValue("delivery_time", GetMetadata("DeliveryTime")); + cmd.Parameters.AddWithValue("conversation_id", GetMetadata("ConversationId")); + + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + public async Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken = default) + { + // Insert SagaSnapshot into saga_snapshots table + var cmd = new NpgsqlCommand(@" + INSERT INTO saga_snapshots ( + id, saga_id, saga_type, start_time, finish_time, status, state_after_change, initiating_message, outgoing_messages, endpoint, processed_at + ) VALUES ( + @id, @saga_id, @saga_type, @start_time, @finish_time, @status, @state_after_change, @initiating_message, @outgoing_messages, @endpoint, @processed_at + ) + ON CONFLICT (id) DO NOTHING;", connection, transaction); + + cmd.Parameters.AddWithValue("id", sagaSnapshot.Id ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("saga_id", sagaSnapshot.SagaId); + cmd.Parameters.AddWithValue("saga_type", sagaSnapshot.SagaType ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("start_time", sagaSnapshot.StartTime); + cmd.Parameters.AddWithValue("finish_time", sagaSnapshot.FinishTime); + cmd.Parameters.AddWithValue("status", sagaSnapshot.Status.ToString()); + cmd.Parameters.AddWithValue("state_after_change", sagaSnapshot.StateAfterChange ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("initiating_message", JsonSerializer.Serialize(sagaSnapshot.InitiatingMessage)); + cmd.Parameters.AddWithValue("outgoing_messages", JsonSerializer.Serialize(sagaSnapshot.OutgoingMessages)); + cmd.Parameters.AddWithValue("endpoint", sagaSnapshot.Endpoint ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("processed_at", sagaSnapshot.ProcessedAt); + + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + public async Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken = default) + { + // Insert KnownEndpoint into known_endpoints table + var cmd = new NpgsqlCommand(@" + INSERT INTO known_endpoints ( + id, name, host_id, host, last_seen + ) VALUES ( + @id, @name, @host_id, @host, @last_seen + ) + ON CONFLICT (id) DO NOTHING;", connection, transaction); + + cmd.Parameters.AddWithValue("id", knownEndpoint.Id ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("name", knownEndpoint.Name ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("host_id", knownEndpoint.HostId); + cmd.Parameters.AddWithValue("host", knownEndpoint.Host ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("last_seen", knownEndpoint.LastSeen); + + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs new file mode 100644 index 0000000000..7b76ed259f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork +{ + using System.Threading; + using System.Threading.Tasks; + using Npgsql; + using ServiceControl.Audit.Persistence.UnitOfWork; + using ServiceControl.Audit.Persistence.PostgreSQL; + + public class PostgreSQLAuditIngestionUnitOfWorkFactory : IAuditIngestionUnitOfWorkFactory + { + readonly PostgreSQLConnectionFactory connectionFactory; + + public PostgreSQLAuditIngestionUnitOfWorkFactory(PostgreSQLConnectionFactory connectionFactory) + { + this.connectionFactory = connectionFactory; + } + + public async ValueTask StartNew(int batchSize, CancellationToken cancellationToken) + { + var connection = await connectionFactory.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + return new PostgreSQLAuditIngestionUnitOfWork(connection, transaction); + } + + public bool CanIngestMore() => true; // TODO: Implement logic based on storage state + } +} diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/persistence.manifest b/src/ServiceControl.Audit.Persistence.PostgreSQL/persistence.manifest new file mode 100644 index 0000000000..8c29a40f00 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/persistence.manifest @@ -0,0 +1,7 @@ +{ + "Name": "PostgreSQL", + "DisplayName": "PostgreSQL", + "Description": "PostgreSQL ServiceControl Audit persister", + "AssemblyName": "ServiceControl.Audit.Persistence.PostgreSQL", + "TypeName": "ServiceControl.Audit.Persistence.PostgreSQL.PostgreSQLPersistenceConfiguration, ServiceControl.Audit.Persistence.PostgreSQL" +} \ No newline at end of file diff --git a/src/ServiceControl.sln b/src/ServiceControl.sln index 5a068daa4d..4a3169f3be 100644 --- a/src/ServiceControl.sln +++ b/src/ServiceControl.sln @@ -188,6 +188,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceControl.Hosting", "S EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SetupProcessFake", "SetupProcessFake\SetupProcessFake.csproj", "{5837F789-69B9-44BE-B114-3A2880F06CAB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceControl.Audit.Persistence.PostgreSQL", "ServiceControl.Audit.Persistence.PostgreSQL\ServiceControl.Audit.Persistence.PostgreSQL.csproj", "{921392AB-D2A9-40DB-A5DE-00D642CE541F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1026,6 +1028,18 @@ Global {5837F789-69B9-44BE-B114-3A2880F06CAB}.Release|x64.Build.0 = Release|Any CPU {5837F789-69B9-44BE-B114-3A2880F06CAB}.Release|x86.ActiveCfg = Release|Any CPU {5837F789-69B9-44BE-B114-3A2880F06CAB}.Release|x86.Build.0 = Release|Any CPU + {921392AB-D2A9-40DB-A5DE-00D642CE541F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {921392AB-D2A9-40DB-A5DE-00D642CE541F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {921392AB-D2A9-40DB-A5DE-00D642CE541F}.Debug|x64.ActiveCfg = Debug|Any CPU + {921392AB-D2A9-40DB-A5DE-00D642CE541F}.Debug|x64.Build.0 = Debug|Any CPU + {921392AB-D2A9-40DB-A5DE-00D642CE541F}.Debug|x86.ActiveCfg = Debug|Any CPU + {921392AB-D2A9-40DB-A5DE-00D642CE541F}.Debug|x86.Build.0 = Debug|Any CPU + {921392AB-D2A9-40DB-A5DE-00D642CE541F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {921392AB-D2A9-40DB-A5DE-00D642CE541F}.Release|Any CPU.Build.0 = Release|Any CPU + {921392AB-D2A9-40DB-A5DE-00D642CE541F}.Release|x64.ActiveCfg = Release|Any CPU + {921392AB-D2A9-40DB-A5DE-00D642CE541F}.Release|x64.Build.0 = Release|Any CPU + {921392AB-D2A9-40DB-A5DE-00D642CE541F}.Release|x86.ActiveCfg = Release|Any CPU + {921392AB-D2A9-40DB-A5DE-00D642CE541F}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1111,6 +1125,7 @@ Global {18DBEEF5-42EE-4C1D-A05B-87B21C067D53} = {E0E45F22-35E3-4AD8-B09E-EFEA5A2F18EE} {481032A1-1106-4C6C-B75E-512F2FB08882} = {9AF9D3C7-E859-451B-BA4D-B954D289213A} {5837F789-69B9-44BE-B114-3A2880F06CAB} = {927A078A-E271-4878-A153-86D71AE510E2} + {921392AB-D2A9-40DB-A5DE-00D642CE541F} = {BD162BC6-705F-45B4-A6B5-C138DC966C1D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3B9E5B72-F580-465A-A22C-2D2148AF4EB4} From 49b3dd1d6d1ff567435a313435219c9ca8768b6c Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 27 Aug 2025 11:44:12 +1000 Subject: [PATCH 05/41] Moving things around --- .../.editorconfig | 4 + .../PostgreSQLAttachmentsBodyStorage.cs | 2 +- .../DatabaseConfiguration.cs | 21 ++++++ .../PostgreSQLAuditDataStore.cs | 2 +- .../PostgreSQLConnectionFactory.cs | 16 +--- .../PostgreSQLFailedAuditStorage.cs | 2 +- .../PostgreSQLPersistence.cs | 7 +- .../PostgreSQLPersistenceConfiguration.cs | 75 ++++--------------- .../PostgreSQLPersistenceInstaller.cs | 75 +++++++++++++++++++ .../PostgreSQLAuditIngestionUnitOfWork.cs | 20 ++--- ...stgreSQLAuditIngestionUnitOfWorkFactory.cs | 7 +- 11 files changed, 139 insertions(+), 92 deletions(-) create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/.editorconfig create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/DatabaseConfiguration.cs create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/.editorconfig b/src/ServiceControl.Audit.Persistence.PostgreSQL/.editorconfig new file mode 100644 index 0000000000..8d96183ebc --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/.editorconfig @@ -0,0 +1,4 @@ +[*.cs] + +# Justification: ServiceControl app has no synchronization context +dotnet_diagnostic.CA2007.severity = none \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/BodyStorage/PostgreSQLAttachmentsBodyStorage.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/BodyStorage/PostgreSQLAttachmentsBodyStorage.cs index 25bea90a25..96ee4d6f37 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/BodyStorage/PostgreSQLAttachmentsBodyStorage.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/BodyStorage/PostgreSQLAttachmentsBodyStorage.cs @@ -5,7 +5,7 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL.BodyStorage using System.Threading.Tasks; using ServiceControl.Audit.Auditing.BodyStorage; - public class PostgreSQLAttachmentsBodyStorage : IBodyStorage + class PostgreSQLAttachmentsBodyStorage : IBodyStorage { public Task Store(string bodyId, string contentType, int bodySize, Stream bodyStream, CancellationToken cancellationToken) => throw new System.NotImplementedException(); public Task TryFetch(string bodyId, CancellationToken cancellationToken) => throw new System.NotImplementedException(); diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/DatabaseConfiguration.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/DatabaseConfiguration.cs new file mode 100644 index 0000000000..9388463dcb --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/DatabaseConfiguration.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Audit.Persistence.PostgreSQL +{ + using System; + + class DatabaseConfiguration( + string databaseName, + int expirationProcessTimerInSeconds, + TimeSpan auditRetentionPeriod, + int maxBodySizeToStore, + string connectionString) + { + public string Name { get; } = databaseName; + + public int ExpirationProcessTimerInSeconds { get; } = expirationProcessTimerInSeconds; + + public TimeSpan AuditRetentionPeriod { get; } = auditRetentionPeriod; + + public int MaxBodySizeToStore { get; } = maxBodySizeToStore; + public string ConnectionString { get; } = connectionString; + } +} diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs index 2694d30c52..4b190d50d2 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs @@ -11,7 +11,7 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL using ServiceControl.Audit.Persistence; using ServiceControl.SagaAudit; - public class PostgreSQLAuditDataStore : IAuditDataStore + class PostgreSQLAuditDataStore : IAuditDataStore { public Task GetMessageBody(string messageId, CancellationToken cancellationToken) => throw new NotImplementedException(); public Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs index a919ca1ce4..957bd7a904 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs @@ -4,24 +4,16 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL using System.Threading.Tasks; using System.Threading; - public class PostgreSQLConnectionFactory + class PostgreSQLConnectionFactory { readonly string connectionString; - public PostgreSQLConnectionFactory(string connectionString) - { - this.connectionString = connectionString; - } - - public NpgsqlConnection CreateConnection() - { - return new NpgsqlConnection(connectionString); - } + public PostgreSQLConnectionFactory(DatabaseConfiguration databaseConfiguration) => connectionString = databaseConfiguration.ConnectionString; - public async Task OpenConnectionAsync(CancellationToken cancellationToken = default) + public async Task OpenConnection(CancellationToken cancellationToken) { var conn = new NpgsqlConnection(connectionString); - await conn.OpenAsync(cancellationToken).ConfigureAwait(false); + await conn.OpenAsync(cancellationToken); return conn; } } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs index 912c4e16b9..12ae17175d 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs @@ -6,7 +6,7 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL using ServiceControl.Audit.Auditing; using ServiceControl.Audit.Persistence; - public class PostgreSQLFailedAuditStorage : IFailedAuditStorage + class PostgreSQLFailedAuditStorage : IFailedAuditStorage { public Task GetFailedAuditsCount() => throw new NotImplementedException(); public Task ProcessFailedMessages(Func, CancellationToken, Task> onMessage, CancellationToken cancellationToken) => throw new NotImplementedException(); diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs index fd3417dcd2..86494b9a19 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs @@ -7,15 +7,18 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL using ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; using ServiceControl.Audit.Persistence.UnitOfWork; - public class PostgreSQLPersistence : IPersistence + class PostgreSQLPersistence(DatabaseConfiguration databaseConfiguration) : IPersistence { public void AddInstaller(IServiceCollection services) { - AddPersistence(services); + services.AddSingleton(databaseConfiguration); + services.AddSingleton(); + services.AddHostedService(); } public void AddPersistence(IServiceCollection services) { + services.AddSingleton(databaseConfiguration); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceConfiguration.cs index 5c6319a495..21c83d5c82 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceConfiguration.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceConfiguration.cs @@ -1,82 +1,35 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL { + using System; using System.Collections.Generic; - using Npgsql; using ServiceControl.Audit.Persistence; public class PostgreSQLPersistenceConfiguration : IPersistenceConfiguration { public string Name => "PostgreSQL"; - public IEnumerable ConfigurationKeys => new[] { "PostgreSqlConnectionString" }; + public IEnumerable ConfigurationKeys => ["PostgreSql/ConnectionString", "PostgreSql/DatabaseName"]; + + const int ExpirationProcessTimerInSecondsDefault = 600; public IPersistence Create(PersistenceSettings settings) { - settings.PersisterSpecificSettings.TryGetValue("PostgreSqlConnectionString", out var connectionString); - using var connection = new NpgsqlConnection(connectionString); - connection.Open(); - - // Create processed_messages table - using (var cmd = new NpgsqlCommand(@" - CREATE TABLE IF NOT EXISTS processed_messages ( - id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - unique_message_id TEXT, - message_metadata JSONB, - headers JSONB, - processed_at TIMESTAMPTZ, - body BYTEA, - message_id TEXT, - message_type TEXT, - is_system_message BOOLEAN, - status TEXT, - time_sent TIMESTAMPTZ, - receiving_endpoint_name TEXT, - critical_time INTERVAL, - processing_time INTERVAL, - delivery_time INTERVAL, - conversation_id TEXT, - query tsvector GENERATED ALWAYS AS ( - setweight(to_tsvector('english', coalesce(headers::text, '')), 'A') || - setweight(to_tsvector('english', coalesce(body::text, '')), 'B') - ) STORED - );", connection)) - { - cmd.ExecuteNonQuery(); - } - - // Create saga_snapshots table - using (var cmd = new NpgsqlCommand(@" - CREATE TABLE IF NOT EXISTS saga_snapshots ( - id TEXT PRIMARY KEY, - saga_id UUID, - saga_type TEXT, - start_time TIMESTAMPTZ, - finish_time TIMESTAMPTZ, - status TEXT, - state_after_change TEXT, - initiating_message JSONB, - outgoing_messages JSONB, - endpoint TEXT, - processed_at TIMESTAMPTZ - );", connection)) + if (!settings.PersisterSpecificSettings.TryGetValue("PostgreSql/ConnectionString", out var connectionString)) { - cmd.ExecuteNonQuery(); + throw new Exception("PostgreSql/ConnectionString is not configured."); } - // Create known_endpoints table - using (var cmd = new NpgsqlCommand(@" - CREATE TABLE IF NOT EXISTS known_endpoints ( - id TEXT PRIMARY KEY, - name TEXT, - host_id UUID, - host TEXT, - last_seen TIMESTAMPTZ - );", connection)) + if (!settings.PersisterSpecificSettings.TryGetValue("PostgreSql/DatabaseName", out var databaseName)) { - cmd.ExecuteNonQuery(); + databaseName = "servicecontrol-audit"; } - return new PostgreSQLPersistence(); + return new PostgreSQLPersistence(new DatabaseConfiguration( + databaseName, + ExpirationProcessTimerInSecondsDefault, + settings.AuditRetentionPeriod, + settings.MaxBodySizeToStore, + connectionString)); } } } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs new file mode 100644 index 0000000000..bb81e1a5a9 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs @@ -0,0 +1,75 @@ +namespace ServiceControl.Audit.Persistence.PostgreSQL +{ + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.Hosting; + using Npgsql; + + class PostgreSQLPersistenceInstaller(PostgreSQLConnectionFactory connectionFactory) : BackgroundService + { + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var connection = await connectionFactory.OpenConnection(stoppingToken); + + // Create processed_messages table + using (var cmd = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS processed_messages ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + unique_message_id TEXT, + message_metadata JSONB, + headers JSONB, + processed_at TIMESTAMPTZ, + body BYTEA, + message_id TEXT, + message_type TEXT, + is_system_message BOOLEAN, + status TEXT, + time_sent TIMESTAMPTZ, + receiving_endpoint_name TEXT, + critical_time INTERVAL, + processing_time INTERVAL, + delivery_time INTERVAL, + conversation_id TEXT, + query tsvector GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(headers::text, '')), 'A') || + setweight(to_tsvector('english', coalesce(body::text, '')), 'B') + ) STORED + );", connection)) + { + await cmd.ExecuteNonQueryAsync(stoppingToken); + } + + // Create saga_snapshots table + using (var cmd = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS saga_snapshots ( + id TEXT PRIMARY KEY, + saga_id UUID, + saga_type TEXT, + start_time TIMESTAMPTZ, + finish_time TIMESTAMPTZ, + status TEXT, + state_after_change TEXT, + initiating_message JSONB, + outgoing_messages JSONB, + endpoint TEXT, + processed_at TIMESTAMPTZ + );", connection)) + { + await cmd.ExecuteNonQueryAsync(stoppingToken); + } + + // Create known_endpoints table + using (var cmd = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS known_endpoints ( + id TEXT PRIMARY KEY, + name TEXT, + host_id UUID, + host TEXT, + last_seen TIMESTAMPTZ + );", connection)) + { + await cmd.ExecuteNonQueryAsync(stoppingToken); + } + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index cdeea30e96..232b0146b0 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -10,7 +10,7 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork using ServiceControl.Audit.Persistence.UnitOfWork; using ServiceControl.SagaAudit; - public class PostgreSQLAuditIngestionUnitOfWork : IAuditIngestionUnitOfWork + class PostgreSQLAuditIngestionUnitOfWork : IAuditIngestionUnitOfWork { readonly NpgsqlConnection connection; readonly NpgsqlTransaction transaction; @@ -23,13 +23,13 @@ public PostgreSQLAuditIngestionUnitOfWork(NpgsqlConnection connection, NpgsqlTra public async ValueTask DisposeAsync() { - await transaction.DisposeAsync().ConfigureAwait(false); - await connection.DisposeAsync().ConfigureAwait(false); + await transaction.DisposeAsync(); + await connection.DisposeAsync(); } - public async Task CompleteAsync(CancellationToken cancellationToken = default) + public async Task CompleteAsync(CancellationToken cancellationToken) { - await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken); } public async Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body, CancellationToken cancellationToken = default) @@ -72,10 +72,10 @@ INSERT INTO processed_messages ( cmd.Parameters.AddWithValue("delivery_time", GetMetadata("DeliveryTime")); cmd.Parameters.AddWithValue("conversation_id", GetMetadata("ConversationId")); - await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(cancellationToken); } - public async Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken = default) + public async Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken) { // Insert SagaSnapshot into saga_snapshots table var cmd = new NpgsqlCommand(@" @@ -98,10 +98,10 @@ INSERT INTO saga_snapshots ( cmd.Parameters.AddWithValue("endpoint", sagaSnapshot.Endpoint ?? (object)DBNull.Value); cmd.Parameters.AddWithValue("processed_at", sagaSnapshot.ProcessedAt); - await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(cancellationToken); } - public async Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken = default) + public async Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken) { // Insert KnownEndpoint into known_endpoints table var cmd = new NpgsqlCommand(@" @@ -118,7 +118,7 @@ INSERT INTO known_endpoints ( cmd.Parameters.AddWithValue("host", knownEndpoint.Host ?? (object)DBNull.Value); cmd.Parameters.AddWithValue("last_seen", knownEndpoint.LastSeen); - await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(cancellationToken); } } } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs index 7b76ed259f..5e3db435d4 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs @@ -2,11 +2,10 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork { using System.Threading; using System.Threading.Tasks; - using Npgsql; using ServiceControl.Audit.Persistence.UnitOfWork; using ServiceControl.Audit.Persistence.PostgreSQL; - public class PostgreSQLAuditIngestionUnitOfWorkFactory : IAuditIngestionUnitOfWorkFactory + class PostgreSQLAuditIngestionUnitOfWorkFactory : IAuditIngestionUnitOfWorkFactory { readonly PostgreSQLConnectionFactory connectionFactory; @@ -17,8 +16,8 @@ public PostgreSQLAuditIngestionUnitOfWorkFactory(PostgreSQLConnectionFactory con public async ValueTask StartNew(int batchSize, CancellationToken cancellationToken) { - var connection = await connectionFactory.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); - var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + var connection = await connectionFactory.OpenConnection(cancellationToken); + var transaction = await connection.BeginTransactionAsync(cancellationToken); return new PostgreSQLAuditIngestionUnitOfWork(connection, transaction); } From 6525de8a98493dcd4464173176e0409e0f826249 Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 27 Aug 2025 14:57:40 +1000 Subject: [PATCH 06/41] Using namespace file scoped --- .../PostgreSQLAttachmentsBodyStorage.cs | 20 +- .../DatabaseConfiguration.cs | 33 ++-- .../PostgreSQLAuditDataStore.cs | 47 +++-- .../PostgreSQLConnectionFactory.cs | 35 ++-- .../PostgreSQLFailedAuditStorage.cs | 24 ++- .../PostgreSQLPersistence.cs | 46 +++-- .../PostgreSQLPersistenceConfiguration.cs | 59 +++--- .../PostgreSQLPersistenceInstaller.cs | 139 +++++++------- .../PostgreSQLAuditIngestionUnitOfWork.cs | 171 +++++++++--------- ...stgreSQLAuditIngestionUnitOfWorkFactory.cs | 40 ++-- 10 files changed, 314 insertions(+), 300 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/BodyStorage/PostgreSQLAttachmentsBodyStorage.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/BodyStorage/PostgreSQLAttachmentsBodyStorage.cs index 96ee4d6f37..b1bb307b6e 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/BodyStorage/PostgreSQLAttachmentsBodyStorage.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/BodyStorage/PostgreSQLAttachmentsBodyStorage.cs @@ -1,13 +1,11 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL.BodyStorage -{ - using System.IO; - using System.Threading; - using System.Threading.Tasks; - using ServiceControl.Audit.Auditing.BodyStorage; +namespace ServiceControl.Audit.Persistence.PostgreSQL.BodyStorage; - class PostgreSQLAttachmentsBodyStorage : IBodyStorage - { - public Task Store(string bodyId, string contentType, int bodySize, Stream bodyStream, CancellationToken cancellationToken) => throw new System.NotImplementedException(); - public Task TryFetch(string bodyId, CancellationToken cancellationToken) => throw new System.NotImplementedException(); - } +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Audit.Auditing.BodyStorage; +class PostgreSQLAttachmentsBodyStorage : IBodyStorage +{ + public Task Store(string bodyId, string contentType, int bodySize, Stream bodyStream, CancellationToken cancellationToken) => throw new System.NotImplementedException(); + public Task TryFetch(string bodyId, CancellationToken cancellationToken) => throw new System.NotImplementedException(); } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/DatabaseConfiguration.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/DatabaseConfiguration.cs index 9388463dcb..41ef7619d2 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/DatabaseConfiguration.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/DatabaseConfiguration.cs @@ -1,21 +1,20 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL -{ - using System; - - class DatabaseConfiguration( - string databaseName, - int expirationProcessTimerInSeconds, - TimeSpan auditRetentionPeriod, - int maxBodySizeToStore, - string connectionString) - { - public string Name { get; } = databaseName; +namespace ServiceControl.Audit.Persistence.PostgreSQL; - public int ExpirationProcessTimerInSeconds { get; } = expirationProcessTimerInSeconds; +using System; +class DatabaseConfiguration( + string databaseName, + string adminDatabaseName, + int expirationProcessTimerInSeconds, + TimeSpan auditRetentionPeriod, + int maxBodySizeToStore, + string connectionString) +{ + public string Name { get; } = databaseName; + public string AdminDatabaseName { get; } = adminDatabaseName; + public int ExpirationProcessTimerInSeconds { get; } = expirationProcessTimerInSeconds; - public TimeSpan AuditRetentionPeriod { get; } = auditRetentionPeriod; + public TimeSpan AuditRetentionPeriod { get; } = auditRetentionPeriod; - public int MaxBodySizeToStore { get; } = maxBodySizeToStore; - public string ConnectionString { get; } = connectionString; - } + public int MaxBodySizeToStore { get; } = maxBodySizeToStore; + public string ConnectionString { get; } = connectionString; } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs index 4b190d50d2..163ea580a9 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs @@ -1,26 +1,25 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL -{ - using System; - using System.Collections.Generic; - using System.Threading; - using System.Threading.Tasks; - using ServiceControl.Audit.Auditing; - using ServiceControl.Audit.Auditing.MessagesView; - using ServiceControl.Audit.Infrastructure; - using ServiceControl.Audit.Monitoring; - using ServiceControl.Audit.Persistence; - using ServiceControl.SagaAudit; +namespace ServiceControl.Audit.Persistence.PostgreSQL; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Audit.Auditing; +using ServiceControl.Audit.Auditing.MessagesView; +using ServiceControl.Audit.Infrastructure; +using ServiceControl.Audit.Monitoring; +using ServiceControl.Audit.Persistence; +using ServiceControl.SagaAudit; - class PostgreSQLAuditDataStore : IAuditDataStore - { - public Task GetMessageBody(string messageId, CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task>> QueryKnownEndpoints(CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task> QuerySagaHistoryById(Guid input, CancellationToken cancellationToken) => throw new NotImplementedException(); - } +class PostgreSQLAuditDataStore : IAuditDataStore +{ + public Task GetMessageBody(string messageId, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task>> QueryKnownEndpoints(CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task> QuerySagaHistoryById(Guid input, CancellationToken cancellationToken) => throw new NotImplementedException(); } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs index 957bd7a904..58a8237035 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs @@ -1,20 +1,25 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL -{ - using Npgsql; - using System.Threading.Tasks; - using System.Threading; +namespace ServiceControl.Audit.Persistence.PostgreSQL; - class PostgreSQLConnectionFactory +using Npgsql; +using System.Threading.Tasks; +using System.Threading; +class PostgreSQLConnectionFactory(DatabaseConfiguration databaseConfiguration) +{ + public async Task OpenConnection(CancellationToken cancellationToken) { - readonly string connectionString; - - public PostgreSQLConnectionFactory(DatabaseConfiguration databaseConfiguration) => connectionString = databaseConfiguration.ConnectionString; + var conn = new NpgsqlConnection(databaseConfiguration.ConnectionString); + await conn.OpenAsync(cancellationToken); + return conn; + } - public async Task OpenConnection(CancellationToken cancellationToken) + public async Task OpenAdminConnection(CancellationToken cancellationToken) + { + var builder = new NpgsqlConnectionStringBuilder(databaseConfiguration.ConnectionString) { - var conn = new NpgsqlConnection(connectionString); - await conn.OpenAsync(cancellationToken); - return conn; - } + Database = databaseConfiguration.AdminDatabaseName + }; + var conn = new NpgsqlConnection(builder.ConnectionString); + await conn.OpenAsync(cancellationToken); + return conn; } -} +} \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs index 12ae17175d..8456e4e406 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs @@ -1,15 +1,13 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL -{ - using System; - using System.Threading; - using System.Threading.Tasks; - using ServiceControl.Audit.Auditing; - using ServiceControl.Audit.Persistence; +namespace ServiceControl.Audit.Persistence.PostgreSQL; - class PostgreSQLFailedAuditStorage : IFailedAuditStorage - { - public Task GetFailedAuditsCount() => throw new NotImplementedException(); - public Task ProcessFailedMessages(Func, CancellationToken, Task> onMessage, CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task SaveFailedAuditImport(FailedAuditImport message) => throw new NotImplementedException(); - } +using System; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Audit.Auditing; +using ServiceControl.Audit.Persistence; +class PostgreSQLFailedAuditStorage : IFailedAuditStorage +{ + public Task GetFailedAuditsCount() => throw new NotImplementedException(); + public Task ProcessFailedMessages(Func, CancellationToken, Task> onMessage, CancellationToken cancellationToken) => throw new NotImplementedException(); + public Task SaveFailedAuditImport(FailedAuditImport message) => throw new NotImplementedException(); } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs index 86494b9a19..b7694ab441 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs @@ -1,29 +1,27 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL -{ - using Microsoft.Extensions.DependencyInjection; - using ServiceControl.Audit.Auditing.BodyStorage; - using ServiceControl.Audit.Persistence; - using ServiceControl.Audit.Persistence.PostgreSQL.BodyStorage; - using ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; - using ServiceControl.Audit.Persistence.UnitOfWork; +namespace ServiceControl.Audit.Persistence.PostgreSQL; - class PostgreSQLPersistence(DatabaseConfiguration databaseConfiguration) : IPersistence +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Audit.Auditing.BodyStorage; +using ServiceControl.Audit.Persistence; +using ServiceControl.Audit.Persistence.PostgreSQL.BodyStorage; +using ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; +using ServiceControl.Audit.Persistence.UnitOfWork; +class PostgreSQLPersistence(DatabaseConfiguration databaseConfiguration) : IPersistence +{ + public void AddInstaller(IServiceCollection services) { - public void AddInstaller(IServiceCollection services) - { - services.AddSingleton(databaseConfiguration); - services.AddSingleton(); - services.AddHostedService(); - } + services.AddSingleton(databaseConfiguration); + services.AddSingleton(); + services.AddHostedService(); + } - public void AddPersistence(IServiceCollection services) - { - services.AddSingleton(databaseConfiguration); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - } + public void AddPersistence(IServiceCollection services) + { + services.AddSingleton(databaseConfiguration); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); } } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceConfiguration.cs index 21c83d5c82..5d28ff08f9 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceConfiguration.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceConfiguration.cs @@ -1,35 +1,42 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL +namespace ServiceControl.Audit.Persistence.PostgreSQL; + +using System; +using System.Collections.Generic; +using Npgsql; +using ServiceControl.Audit.Persistence; + +public class PostgreSQLPersistenceConfiguration : IPersistenceConfiguration { - using System; - using System.Collections.Generic; - using ServiceControl.Audit.Persistence; + public string Name => "PostgreSQL"; - public class PostgreSQLPersistenceConfiguration : IPersistenceConfiguration - { - public string Name => "PostgreSQL"; + public IEnumerable ConfigurationKeys => ["PostgreSql/ConnectionString", "PostgreSql/DatabaseName"]; - public IEnumerable ConfigurationKeys => ["PostgreSql/ConnectionString", "PostgreSql/DatabaseName"]; + const int ExpirationProcessTimerInSecondsDefault = 600; + + public IPersistence Create(PersistenceSettings settings) + { + if (!settings.PersisterSpecificSettings.TryGetValue("PostgreSql/ConnectionString", out var connectionString)) + { + throw new Exception("PostgreSql/ConnectionString is not configured."); + } - const int ExpirationProcessTimerInSecondsDefault = 600; + var builder = new NpgsqlConnectionStringBuilder(connectionString); - public IPersistence Create(PersistenceSettings settings) + if (settings.PersisterSpecificSettings.TryGetValue("PostgreSql/DatabaseName", out var databaseName)) { - if (!settings.PersisterSpecificSettings.TryGetValue("PostgreSql/ConnectionString", out var connectionString)) - { - throw new Exception("PostgreSql/ConnectionString is not configured."); - } - - if (!settings.PersisterSpecificSettings.TryGetValue("PostgreSql/DatabaseName", out var databaseName)) - { - databaseName = "servicecontrol-audit"; - } - - return new PostgreSQLPersistence(new DatabaseConfiguration( - databaseName, - ExpirationProcessTimerInSecondsDefault, - settings.AuditRetentionPeriod, - settings.MaxBodySizeToStore, - connectionString)); + builder.Database = databaseName; } + + settings.PersisterSpecificSettings.TryGetValue("PostgreSql/AdminDatabaseName", out var adminDatabaseName); + + builder.Database ??= "servicecontrol-audit"; + + return new PostgreSQLPersistence(new DatabaseConfiguration( + builder.Database, + adminDatabaseName ?? "postgres", + ExpirationProcessTimerInSecondsDefault, + settings.AuditRetentionPeriod, + settings.MaxBodySizeToStore, + connectionString)); } } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs index bb81e1a5a9..d1f921908b 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs @@ -1,75 +1,88 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL -{ - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Extensions.Hosting; - using Npgsql; - class PostgreSQLPersistenceInstaller(PostgreSQLConnectionFactory connectionFactory) : BackgroundService +namespace ServiceControl.Audit.Persistence.PostgreSQL; + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Npgsql; + +class PostgreSQLPersistenceInstaller(DatabaseConfiguration databaseConfiguration, PostgreSQLConnectionFactory connectionFactory) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - using var connection = await connectionFactory.OpenConnection(stoppingToken); + using var connection = await connectionFactory.OpenConnection(stoppingToken); - // Create processed_messages table - using (var cmd = new NpgsqlCommand(@" - CREATE TABLE IF NOT EXISTS processed_messages ( - id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - unique_message_id TEXT, - message_metadata JSONB, - headers JSONB, - processed_at TIMESTAMPTZ, - body BYTEA, - message_id TEXT, - message_type TEXT, - is_system_message BOOLEAN, - status TEXT, - time_sent TIMESTAMPTZ, - receiving_endpoint_name TEXT, - critical_time INTERVAL, - processing_time INTERVAL, - delivery_time INTERVAL, - conversation_id TEXT, - query tsvector GENERATED ALWAYS AS ( - setweight(to_tsvector('english', coalesce(headers::text, '')), 'A') || - setweight(to_tsvector('english', coalesce(body::text, '')), 'B') - ) STORED - );", connection)) + using (var cmd = new NpgsqlCommand($"SELECT 1 FROM pg_database WHERE datname = @dbname", connection)) + { + cmd.Parameters.AddWithValue("@dbname", databaseConfiguration.Name); + var exists = await cmd.ExecuteScalarAsync(stoppingToken); + if (exists == null) { - await cmd.ExecuteNonQueryAsync(stoppingToken); + using var createCmd = new NpgsqlCommand($"CREATE DATABASE \"{databaseConfiguration.Name}\"", connection); + await createCmd.ExecuteNonQueryAsync(stoppingToken); } + } - // Create saga_snapshots table - using (var cmd = new NpgsqlCommand(@" - CREATE TABLE IF NOT EXISTS saga_snapshots ( - id TEXT PRIMARY KEY, - saga_id UUID, - saga_type TEXT, - start_time TIMESTAMPTZ, - finish_time TIMESTAMPTZ, + // Create processed_messages table + using (var cmd = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS processed_messages ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + unique_message_id TEXT, + message_metadata JSONB, + headers JSONB, + processed_at TIMESTAMPTZ, + body BYTEA, + message_id TEXT, + message_type TEXT, + is_system_message BOOLEAN, status TEXT, - state_after_change TEXT, - initiating_message JSONB, - outgoing_messages JSONB, - endpoint TEXT, - processed_at TIMESTAMPTZ + time_sent TIMESTAMPTZ, + receiving_endpoint_name TEXT, + critical_time INTERVAL, + processing_time INTERVAL, + delivery_time INTERVAL, + conversation_id TEXT, + query tsvector GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(headers::text, '')), 'A') || + setweight(to_tsvector('english', coalesce(body::text, '')), 'B') + ) STORED );", connection)) - { - await cmd.ExecuteNonQueryAsync(stoppingToken); - } + { + await cmd.ExecuteNonQueryAsync(stoppingToken); + } - // Create known_endpoints table - using (var cmd = new NpgsqlCommand(@" - CREATE TABLE IF NOT EXISTS known_endpoints ( - id TEXT PRIMARY KEY, - name TEXT, - host_id UUID, - host TEXT, - last_seen TIMESTAMPTZ - );", connection)) - { - await cmd.ExecuteNonQueryAsync(stoppingToken); - } + // Create saga_snapshots table + using (var cmd = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS saga_snapshots ( + id TEXT PRIMARY KEY, + saga_id UUID, + saga_type TEXT, + start_time TIMESTAMPTZ, + finish_time TIMESTAMPTZ, + status TEXT, + state_after_change TEXT, + initiating_message JSONB, + outgoing_messages JSONB, + endpoint TEXT, + processed_at TIMESTAMPTZ + );", connection)) + { + await cmd.ExecuteNonQueryAsync(stoppingToken); + } + + // Create known_endpoints table + using (var cmd = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS known_endpoints ( + id TEXT PRIMARY KEY, + name TEXT, + host_id UUID, + host TEXT, + last_seen TIMESTAMPTZ + );", connection)) + { + await cmd.ExecuteNonQueryAsync(stoppingToken); } } } + + diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index 232b0146b0..ccbc7ce805 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -1,43 +1,43 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork +namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; + +using System; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using ServiceControl.Audit.Auditing; +using ServiceControl.Audit.Persistence.Monitoring; +using ServiceControl.Audit.Persistence.UnitOfWork; +using ServiceControl.SagaAudit; + +class PostgreSQLAuditIngestionUnitOfWork : IAuditIngestionUnitOfWork { - using System; - using System.Text.Json; - using System.Threading; - using System.Threading.Tasks; - using Npgsql; - using ServiceControl.Audit.Auditing; - using ServiceControl.Audit.Persistence.Monitoring; - using ServiceControl.Audit.Persistence.UnitOfWork; - using ServiceControl.SagaAudit; + readonly NpgsqlConnection connection; + readonly NpgsqlTransaction transaction; - class PostgreSQLAuditIngestionUnitOfWork : IAuditIngestionUnitOfWork + public PostgreSQLAuditIngestionUnitOfWork(NpgsqlConnection connection, NpgsqlTransaction transaction) { - readonly NpgsqlConnection connection; - readonly NpgsqlTransaction transaction; - - public PostgreSQLAuditIngestionUnitOfWork(NpgsqlConnection connection, NpgsqlTransaction transaction) - { - this.connection = connection; - this.transaction = transaction; - } + this.connection = connection; + this.transaction = transaction; + } - public async ValueTask DisposeAsync() - { - await transaction.DisposeAsync(); - await connection.DisposeAsync(); - } + public async ValueTask DisposeAsync() + { + await transaction.DisposeAsync(); + await connection.DisposeAsync(); + } - public async Task CompleteAsync(CancellationToken cancellationToken) - { - await transaction.CommitAsync(cancellationToken); - } + public async Task CompleteAsync(CancellationToken cancellationToken) + { + await transaction.CommitAsync(cancellationToken); + } - public async Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body, CancellationToken cancellationToken = default) - { - object GetMetadata(string key) => processedMessage.MessageMetadata.TryGetValue(key, out var value) ? value ?? DBNull.Value : DBNull.Value; + public async Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body, CancellationToken cancellationToken = default) + { + object GetMetadata(string key) => processedMessage.MessageMetadata.TryGetValue(key, out var value) ? value ?? DBNull.Value : DBNull.Value; - // Insert ProcessedMessage into processed_messages table - var cmd = new NpgsqlCommand(@" + // Insert ProcessedMessage into processed_messages table + var cmd = new NpgsqlCommand(@" INSERT INTO processed_messages ( unique_message_id, message_metadata, headers, processed_at, body, message_id, message_type, is_system_message, status, time_sent, receiving_endpoint_name, @@ -49,36 +49,36 @@ INSERT INTO processed_messages ( ) ;", connection, transaction); - processedMessage.MessageMetadata["ContentLength"] = body.Length; - if (!body.IsEmpty) - { - cmd.Parameters.AddWithValue("body", body); - } - else - { - cmd.Parameters.AddWithValue("body", DBNull.Value); - } - cmd.Parameters.AddWithValue("unique_message_id", processedMessage.UniqueMessageId ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("message_metadata", JsonSerializer.Serialize(processedMessage.MessageMetadata)); - cmd.Parameters.AddWithValue("headers", JsonSerializer.Serialize(processedMessage.Headers)); - cmd.Parameters.AddWithValue("processed_at", processedMessage.ProcessedAt); - cmd.Parameters.AddWithValue("message_id", GetMetadata("MessageId")); - cmd.Parameters.AddWithValue("message_type", GetMetadata("MessageType")); - cmd.Parameters.AddWithValue("is_system_message", GetMetadata("IsSystemMessage")); - cmd.Parameters.AddWithValue("time_sent", GetMetadata("TimeSent")); - cmd.Parameters.AddWithValue("receiving_endpoint_name", GetMetadata("ReceivingEndpoint")); - cmd.Parameters.AddWithValue("critical_time", GetMetadata("CriticalTime")); - cmd.Parameters.AddWithValue("processing_time", GetMetadata("ProcessingTime")); - cmd.Parameters.AddWithValue("delivery_time", GetMetadata("DeliveryTime")); - cmd.Parameters.AddWithValue("conversation_id", GetMetadata("ConversationId")); - - await cmd.ExecuteNonQueryAsync(cancellationToken); + processedMessage.MessageMetadata["ContentLength"] = body.Length; + if (!body.IsEmpty) + { + cmd.Parameters.AddWithValue("body", body); } - - public async Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken) + else { - // Insert SagaSnapshot into saga_snapshots table - var cmd = new NpgsqlCommand(@" + cmd.Parameters.AddWithValue("body", DBNull.Value); + } + cmd.Parameters.AddWithValue("unique_message_id", processedMessage.UniqueMessageId ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("message_metadata", JsonSerializer.Serialize(processedMessage.MessageMetadata)); + cmd.Parameters.AddWithValue("headers", JsonSerializer.Serialize(processedMessage.Headers)); + cmd.Parameters.AddWithValue("processed_at", processedMessage.ProcessedAt); + cmd.Parameters.AddWithValue("message_id", GetMetadata("MessageId")); + cmd.Parameters.AddWithValue("message_type", GetMetadata("MessageType")); + cmd.Parameters.AddWithValue("is_system_message", GetMetadata("IsSystemMessage")); + cmd.Parameters.AddWithValue("time_sent", GetMetadata("TimeSent")); + cmd.Parameters.AddWithValue("receiving_endpoint_name", GetMetadata("ReceivingEndpoint")); + cmd.Parameters.AddWithValue("critical_time", GetMetadata("CriticalTime")); + cmd.Parameters.AddWithValue("processing_time", GetMetadata("ProcessingTime")); + cmd.Parameters.AddWithValue("delivery_time", GetMetadata("DeliveryTime")); + cmd.Parameters.AddWithValue("conversation_id", GetMetadata("ConversationId")); + + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + public async Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken) + { + // Insert SagaSnapshot into saga_snapshots table + var cmd = new NpgsqlCommand(@" INSERT INTO saga_snapshots ( id, saga_id, saga_type, start_time, finish_time, status, state_after_change, initiating_message, outgoing_messages, endpoint, processed_at ) VALUES ( @@ -86,25 +86,25 @@ INSERT INTO saga_snapshots ( ) ON CONFLICT (id) DO NOTHING;", connection, transaction); - cmd.Parameters.AddWithValue("id", sagaSnapshot.Id ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("saga_id", sagaSnapshot.SagaId); - cmd.Parameters.AddWithValue("saga_type", sagaSnapshot.SagaType ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("start_time", sagaSnapshot.StartTime); - cmd.Parameters.AddWithValue("finish_time", sagaSnapshot.FinishTime); - cmd.Parameters.AddWithValue("status", sagaSnapshot.Status.ToString()); - cmd.Parameters.AddWithValue("state_after_change", sagaSnapshot.StateAfterChange ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("initiating_message", JsonSerializer.Serialize(sagaSnapshot.InitiatingMessage)); - cmd.Parameters.AddWithValue("outgoing_messages", JsonSerializer.Serialize(sagaSnapshot.OutgoingMessages)); - cmd.Parameters.AddWithValue("endpoint", sagaSnapshot.Endpoint ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("processed_at", sagaSnapshot.ProcessedAt); + cmd.Parameters.AddWithValue("id", sagaSnapshot.Id ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("saga_id", sagaSnapshot.SagaId); + cmd.Parameters.AddWithValue("saga_type", sagaSnapshot.SagaType ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("start_time", sagaSnapshot.StartTime); + cmd.Parameters.AddWithValue("finish_time", sagaSnapshot.FinishTime); + cmd.Parameters.AddWithValue("status", sagaSnapshot.Status.ToString()); + cmd.Parameters.AddWithValue("state_after_change", sagaSnapshot.StateAfterChange ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("initiating_message", JsonSerializer.Serialize(sagaSnapshot.InitiatingMessage)); + cmd.Parameters.AddWithValue("outgoing_messages", JsonSerializer.Serialize(sagaSnapshot.OutgoingMessages)); + cmd.Parameters.AddWithValue("endpoint", sagaSnapshot.Endpoint ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("processed_at", sagaSnapshot.ProcessedAt); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } + await cmd.ExecuteNonQueryAsync(cancellationToken); + } - public async Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken) - { - // Insert KnownEndpoint into known_endpoints table - var cmd = new NpgsqlCommand(@" + public async Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken) + { + // Insert KnownEndpoint into known_endpoints table + var cmd = new NpgsqlCommand(@" INSERT INTO known_endpoints ( id, name, host_id, host, last_seen ) VALUES ( @@ -112,13 +112,12 @@ INSERT INTO known_endpoints ( ) ON CONFLICT (id) DO NOTHING;", connection, transaction); - cmd.Parameters.AddWithValue("id", knownEndpoint.Id ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("name", knownEndpoint.Name ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("host_id", knownEndpoint.HostId); - cmd.Parameters.AddWithValue("host", knownEndpoint.Host ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("last_seen", knownEndpoint.LastSeen); + cmd.Parameters.AddWithValue("id", knownEndpoint.Id ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("name", knownEndpoint.Name ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("host_id", knownEndpoint.HostId); + cmd.Parameters.AddWithValue("host", knownEndpoint.Host ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("last_seen", knownEndpoint.LastSeen); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } + await cmd.ExecuteNonQueryAsync(cancellationToken); } -} +} \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs index 5e3db435d4..25649fb48c 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs @@ -1,26 +1,24 @@ -namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork +namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; + +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Audit.Persistence.UnitOfWork; +using ServiceControl.Audit.Persistence.PostgreSQL; +class PostgreSQLAuditIngestionUnitOfWorkFactory : IAuditIngestionUnitOfWorkFactory { - using System.Threading; - using System.Threading.Tasks; - using ServiceControl.Audit.Persistence.UnitOfWork; - using ServiceControl.Audit.Persistence.PostgreSQL; + readonly PostgreSQLConnectionFactory connectionFactory; - class PostgreSQLAuditIngestionUnitOfWorkFactory : IAuditIngestionUnitOfWorkFactory + public PostgreSQLAuditIngestionUnitOfWorkFactory(PostgreSQLConnectionFactory connectionFactory) { - readonly PostgreSQLConnectionFactory connectionFactory; - - public PostgreSQLAuditIngestionUnitOfWorkFactory(PostgreSQLConnectionFactory connectionFactory) - { - this.connectionFactory = connectionFactory; - } - - public async ValueTask StartNew(int batchSize, CancellationToken cancellationToken) - { - var connection = await connectionFactory.OpenConnection(cancellationToken); - var transaction = await connection.BeginTransactionAsync(cancellationToken); - return new PostgreSQLAuditIngestionUnitOfWork(connection, transaction); - } + this.connectionFactory = connectionFactory; + } - public bool CanIngestMore() => true; // TODO: Implement logic based on storage state + public async ValueTask StartNew(int batchSize, CancellationToken cancellationToken) + { + var connection = await connectionFactory.OpenConnection(cancellationToken); + var transaction = await connection.BeginTransactionAsync(cancellationToken); + return new PostgreSQLAuditIngestionUnitOfWork(connection, transaction); } -} + + public bool CanIngestMore() => true; +} \ No newline at end of file From bf647825a956294c38312dcc069b213f39b8f72c Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 27 Aug 2025 16:02:49 +1000 Subject: [PATCH 07/41] We have something working --- .../PostgreSQLFailedAuditStorage.cs | 2 +- .../PostgreSQLPersistenceInstaller.cs | 28 +++++++++------ .../PostgreSQLAuditIngestionUnitOfWork.cs | 34 +++++++++---------- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs index 8456e4e406..bacdb4f890 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLFailedAuditStorage.cs @@ -7,7 +7,7 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; using ServiceControl.Audit.Persistence; class PostgreSQLFailedAuditStorage : IFailedAuditStorage { - public Task GetFailedAuditsCount() => throw new NotImplementedException(); + public Task GetFailedAuditsCount() => Task.FromResult(0); public Task ProcessFailedMessages(Func, CancellationToken, Task> onMessage, CancellationToken cancellationToken) => throw new NotImplementedException(); public Task SaveFailedAuditImport(FailedAuditImport message) => throw new NotImplementedException(); } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs index d1f921908b..143b478dee 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs @@ -6,23 +6,24 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; using Microsoft.Extensions.Hosting; using Npgsql; -class PostgreSQLPersistenceInstaller(DatabaseConfiguration databaseConfiguration, PostgreSQLConnectionFactory connectionFactory) : BackgroundService +class PostgreSQLPersistenceInstaller(DatabaseConfiguration databaseConfiguration, PostgreSQLConnectionFactory connectionFactory) : IHostedService { - protected override async Task ExecuteAsync(CancellationToken stoppingToken) + public async Task StartAsync(CancellationToken cancellationToken) { - using var connection = await connectionFactory.OpenConnection(stoppingToken); + using var adminConnection = await connectionFactory.OpenAdminConnection(cancellationToken); - using (var cmd = new NpgsqlCommand($"SELECT 1 FROM pg_database WHERE datname = @dbname", connection)) + using (var cmd = new NpgsqlCommand($"SELECT 1 FROM pg_database WHERE datname = @dbname", adminConnection)) { cmd.Parameters.AddWithValue("@dbname", databaseConfiguration.Name); - var exists = await cmd.ExecuteScalarAsync(stoppingToken); + var exists = await cmd.ExecuteScalarAsync(cancellationToken); if (exists == null) { - using var createCmd = new NpgsqlCommand($"CREATE DATABASE \"{databaseConfiguration.Name}\"", connection); - await createCmd.ExecuteNonQueryAsync(stoppingToken); + using var createCmd = new NpgsqlCommand($"CREATE DATABASE \"{databaseConfiguration.Name}\"", adminConnection); + await createCmd.ExecuteNonQueryAsync(cancellationToken); } } + using var connection = await connectionFactory.OpenConnection(cancellationToken); // Create processed_messages table using (var cmd = new NpgsqlCommand(@" CREATE TABLE IF NOT EXISTS processed_messages ( @@ -35,7 +36,7 @@ CREATE TABLE IF NOT EXISTS processed_messages ( message_id TEXT, message_type TEXT, is_system_message BOOLEAN, - status TEXT, + status NUMERIC, time_sent TIMESTAMPTZ, receiving_endpoint_name TEXT, critical_time INTERVAL, @@ -48,7 +49,7 @@ query tsvector GENERATED ALWAYS AS ( ) STORED );", connection)) { - await cmd.ExecuteNonQueryAsync(stoppingToken); + await cmd.ExecuteNonQueryAsync(cancellationToken); } // Create saga_snapshots table @@ -67,7 +68,7 @@ CREATE TABLE IF NOT EXISTS saga_snapshots ( processed_at TIMESTAMPTZ );", connection)) { - await cmd.ExecuteNonQueryAsync(stoppingToken); + await cmd.ExecuteNonQueryAsync(cancellationToken); } // Create known_endpoints table @@ -80,9 +81,14 @@ CREATE TABLE IF NOT EXISTS known_endpoints ( last_seen TIMESTAMPTZ );", connection)) { - await cmd.ExecuteNonQueryAsync(stoppingToken); + await cmd.ExecuteNonQueryAsync(cancellationToken); } } + + public Task StopAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index ccbc7ce805..3c1c5b2743 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -6,6 +6,7 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; using System.Threading.Tasks; using Npgsql; using ServiceControl.Audit.Auditing; +using ServiceControl.Audit.Monitoring; using ServiceControl.Audit.Persistence.Monitoring; using ServiceControl.Audit.Persistence.UnitOfWork; using ServiceControl.SagaAudit; @@ -23,18 +24,14 @@ public PostgreSQLAuditIngestionUnitOfWork(NpgsqlConnection connection, NpgsqlTra public async ValueTask DisposeAsync() { + await transaction.CommitAsync(); await transaction.DisposeAsync(); await connection.DisposeAsync(); } - public async Task CompleteAsync(CancellationToken cancellationToken) + public async Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body, CancellationToken cancellationToken) { - await transaction.CommitAsync(cancellationToken); - } - - public async Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body, CancellationToken cancellationToken = default) - { - object GetMetadata(string key) => processedMessage.MessageMetadata.TryGetValue(key, out var value) ? value ?? DBNull.Value : DBNull.Value; + T GetMetadata(string key) => processedMessage.MessageMetadata.TryGetValue(key, out var value) ? (T)value ?? default : default; // Insert ProcessedMessage into processed_messages table var cmd = new NpgsqlCommand(@" @@ -59,18 +56,19 @@ INSERT INTO processed_messages ( cmd.Parameters.AddWithValue("body", DBNull.Value); } cmd.Parameters.AddWithValue("unique_message_id", processedMessage.UniqueMessageId ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("message_metadata", JsonSerializer.Serialize(processedMessage.MessageMetadata)); - cmd.Parameters.AddWithValue("headers", JsonSerializer.Serialize(processedMessage.Headers)); + cmd.Parameters.AddWithValue("message_metadata", JsonSerializer.SerializeToDocument(processedMessage.MessageMetadata)); + cmd.Parameters.AddWithValue("headers", JsonSerializer.SerializeToDocument(processedMessage.Headers)); cmd.Parameters.AddWithValue("processed_at", processedMessage.ProcessedAt); - cmd.Parameters.AddWithValue("message_id", GetMetadata("MessageId")); - cmd.Parameters.AddWithValue("message_type", GetMetadata("MessageType")); - cmd.Parameters.AddWithValue("is_system_message", GetMetadata("IsSystemMessage")); - cmd.Parameters.AddWithValue("time_sent", GetMetadata("TimeSent")); - cmd.Parameters.AddWithValue("receiving_endpoint_name", GetMetadata("ReceivingEndpoint")); - cmd.Parameters.AddWithValue("critical_time", GetMetadata("CriticalTime")); - cmd.Parameters.AddWithValue("processing_time", GetMetadata("ProcessingTime")); - cmd.Parameters.AddWithValue("delivery_time", GetMetadata("DeliveryTime")); - cmd.Parameters.AddWithValue("conversation_id", GetMetadata("ConversationId")); + cmd.Parameters.AddWithValue("message_id", GetMetadata("MessageId")); + cmd.Parameters.AddWithValue("message_type", GetMetadata("MessageType")); + cmd.Parameters.AddWithValue("is_system_message", GetMetadata("IsSystemMessage")); + cmd.Parameters.AddWithValue("time_sent", GetMetadata("TimeSent")); + cmd.Parameters.AddWithValue("receiving_endpoint_name", GetMetadata("ReceivingEndpoint").Name); + cmd.Parameters.AddWithValue("critical_time", GetMetadata("CriticalTime")); + cmd.Parameters.AddWithValue("processing_time", GetMetadata("ProcessingTime")); + cmd.Parameters.AddWithValue("delivery_time", GetMetadata("DeliveryTime")); + cmd.Parameters.AddWithValue("conversation_id", GetMetadata("ConversationId")); + cmd.Parameters.AddWithValue("status", (int)(GetMetadata("IsRetried") ? MessageStatus.ResolvedSuccessfully : MessageStatus.Successful)); await cmd.ExecuteNonQueryAsync(cancellationToken); } From 7adb46d9d1caff83c8185f4ca0ca4b380660d412 Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 27 Aug 2025 16:43:01 +1000 Subject: [PATCH 08/41] Adding index and trigger --- .../PostgreSQLPersistenceInstaller.cs | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs index 143b478dee..8036c8c598 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs @@ -43,15 +43,45 @@ CREATE TABLE IF NOT EXISTS processed_messages ( processing_time INTERVAL, delivery_time INTERVAL, conversation_id TEXT, - query tsvector GENERATED ALWAYS AS ( - setweight(to_tsvector('english', coalesce(headers::text, '')), 'A') || - setweight(to_tsvector('english', coalesce(body::text, '')), 'B') - ) STORED + query tsvector );", connection)) { await cmd.ExecuteNonQueryAsync(cancellationToken); } + // Create trigger for full text search + using (var cmd = new NpgsqlCommand(@" + CREATE OR REPLACE FUNCTION processed_messages_tsvector_update() RETURNS trigger AS $$ + BEGIN + NEW.query := + setweight(to_tsvector('english', coalesce(NEW.headers::text, '')), 'A') || + setweight(to_tsvector('english', coalesce(convert_from(NEW.body, 'UTF8'), '')), 'B'); + RETURN NEW; + END + $$ LANGUAGE plpgsql; + + CREATE TRIGGER processed_messages_tsvector_trigger + BEFORE INSERT OR UPDATE ON processed_messages + FOR EACH ROW EXECUTE FUNCTION processed_messages_tsvector_update();", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + // Create index on processed_messages for specified columns + using (var cmd = new NpgsqlCommand(@" + CREATE INDEX IF NOT EXISTS idx_processed_messages_multi ON processed_messages ( + message_id, + time_sent, + receiving_endpoint_name, + critical_time, + processing_time, + delivery_time, + conversation_id, + is_system_message + );", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + // Create saga_snapshots table using (var cmd = new NpgsqlCommand(@" CREATE TABLE IF NOT EXISTS saga_snapshots ( From 99e8c61dd3466ee15adc18868e1bace09de4bc9c Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 28 Aug 2025 11:30:13 +1000 Subject: [PATCH 09/41] Returning results --- .../PostgreSQLAuditDataStore.cs | 222 +++++++++++++++++- .../PostgreSQLConnectionFactory.cs | 10 +- .../PostgreSQLPersistenceInstaller.cs | 1 + ...ontrol.Audit.Persistence.PostgreSQL.csproj | 1 + 4 files changed, 225 insertions(+), 9 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs index 163ea580a9..4b74703d1c 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs @@ -2,8 +2,12 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; using System; using System.Collections.Generic; +using System.Text; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using Npgsql; +using NServiceBus; using ServiceControl.Audit.Auditing; using ServiceControl.Audit.Auditing.MessagesView; using ServiceControl.Audit.Infrastructure; @@ -11,15 +15,219 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; using ServiceControl.Audit.Persistence; using ServiceControl.SagaAudit; -class PostgreSQLAuditDataStore : IAuditDataStore +class PostgreSQLAuditDataStore(PostgreSQLConnectionFactory connectionFactory) : IAuditDataStore { - public Task GetMessageBody(string messageId, CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public async Task GetMessageBody(string messageId, CancellationToken cancellationToken) + { + using var conn = await connectionFactory.OpenConnection(cancellationToken); + + using var cmd = new NpgsqlCommand(@" + select body, headers from processed_messages + where message_id = @message_id + LIMIT 1;", conn); + + cmd.Parameters.AddWithValue("message_id", messageId); + + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + if (await reader.ReadAsync(cancellationToken)) + { + var stream = await reader.GetStreamAsync(reader.GetOrdinal("body"), cancellationToken); + var contentType = reader.GetFieldValue>(reader.GetOrdinal("headers")).GetValueOrDefault(Headers.ContentType, "text/xml"); + + return MessageBodyView.FromStream(stream, contentType, (int)stream.Length, string.Empty); + } + + return MessageBodyView.NotFound(); + } + + async Task>> GetAllMessages( + string? conversationId, bool? includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, + string? endpointName, + string? q, + CancellationToken cancellationToken) + { + using var conn = await connectionFactory.OpenConnection(cancellationToken); + + var sql = new StringBuilder(@"select unique_message_id, + message_metadata, + headers, + processed_at, + message_id, + message_type, + is_system_message, + status, + time_sent, + receiving_endpoint_name, + critical_time, + processing_time, + delivery_time, + conversation_id from processed_messages + where 1 = 1"); + + if (includeSystemMessages.HasValue) + { + sql.Append(" and is_system_message = @is_system_message"); + } + + if (!string.IsNullOrWhiteSpace(q)) + { + sql.Append(" and query @@ to_tsquery('english', @search)"); + } + + if (!string.IsNullOrWhiteSpace(conversationId)) + { + sql.Append(" and conversation_id = @conversation_id"); + } + + if (!string.IsNullOrWhiteSpace(endpointName)) + { + sql.Append(" and receiving_endpoint_name = @endpoint_name"); + } + + if (timeSentRange?.From != null) + { + sql.Append(" and time_sent >= @time_sent_start"); + } + + if (timeSentRange?.To != null) + { + sql.Append(" and time_sent <= @time_sent_end"); + } + + sql.Append(" ORDER BY"); + switch (sortInfo.Sort) + { + + case "id": + case "message_id": + sql.Append(" message_id"); + break; + case "message_type": + sql.Append(" message_type"); + break; + case "critical_time": + sql.Append(" critical_time"); + break; + case "delivery_time": + sql.Append(" delivery_time"); + break; + case "processing_time": + sql.Append(" processing_time"); + break; + case "processed_at": + sql.Append(" processed_at"); + break; + case "status": + sql.Append(" status"); + break; + default: + sql.Append(" time_sent"); + break; + } + + if (sortInfo.Direction == "asc") + { + sql.Append(" ASC"); + } + else + { + sql.Append(" DESC"); + } + + sql.Append($" LIMIT {pagingInfo.PageSize} OFFSET {pagingInfo.Offset};"); + + var query = sql.ToString(); + using var cmd = new NpgsqlCommand(query, conn); + + if (!string.IsNullOrWhiteSpace(q)) + { + cmd.Parameters.AddWithValue("search", q); + } + if (!string.IsNullOrWhiteSpace(endpointName)) + { + cmd.Parameters.AddWithValue("endpoint_name", endpointName); + } + if (!string.IsNullOrWhiteSpace(conversationId)) + { + cmd.Parameters.AddWithValue("conversation_id", conversationId); + } + if (includeSystemMessages.HasValue) + { + cmd.Parameters.AddWithValue("is_system_message", includeSystemMessages); + } + if (timeSentRange?.From != null) + { + cmd.Parameters.AddWithValue("time_sent_start", timeSentRange.From); + } + if (timeSentRange?.To != null) + { + cmd.Parameters.AddWithValue("time_sent_end", timeSentRange.To); + } + + return await ReturnResults(cmd, cancellationToken); + } + + public async Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken) + { + return await GetAllMessages(null, includeSystemMessages, pagingInfo, sortInfo, timeSentRange, null, null, cancellationToken); + } + public Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken) => throw new NotImplementedException(); public Task>> QueryKnownEndpoints(CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) + { + return GetAllMessages(null, null, pagingInfo, sortInfo, timeSentRange, searchParam, null, cancellationToken); + } + public async Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) + { + return await GetAllMessages(conversationId, null, pagingInfo, sortInfo, null, null, null, cancellationToken); + } + + public async Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) + { + return await GetAllMessages(null, includeSystemMessages, pagingInfo, sortInfo, timeSentRange, null, endpointName, cancellationToken); + } + + public Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) + { + return GetAllMessages(null, null, pagingInfo, sortInfo, timeSentRange, keyword, endpoint, cancellationToken); + } public Task> QuerySagaHistoryById(Guid input, CancellationToken cancellationToken) => throw new NotImplementedException(); + + async Task>> ReturnResults(NpgsqlCommand cmd, CancellationToken cancellationToken = default) + { + var results = new List(); + + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + var headers = reader.GetFieldValue>(reader.GetOrdinal("headers")); + var messageMetadata = reader.GetFieldValue>(reader.GetOrdinal("message_metadata")); + + results.Add(new MessagesView + { + Id = reader.GetFieldValue(reader.GetOrdinal("unique_message_id")), + MessageId = reader.GetFieldValue(reader.GetOrdinal("message_id")), + MessageType = reader.GetFieldValue(reader.GetOrdinal("message_type")), + SendingEndpoint = JsonSerializer.Deserialize((JsonElement)messageMetadata["SendingEndpoint"]), + ReceivingEndpoint = JsonSerializer.Deserialize((JsonElement)messageMetadata["ReceivingEndpoint"]), + TimeSent = reader.GetFieldValue(reader.GetOrdinal("time_sent")), + ProcessedAt = reader.GetFieldValue(reader.GetOrdinal("processed_at")), + CriticalTime = reader.GetFieldValue(reader.GetOrdinal("critical_time")), + ProcessingTime = reader.GetFieldValue(reader.GetOrdinal("processing_time")), + DeliveryTime = reader.GetFieldValue(reader.GetOrdinal("delivery_time")), + IsSystemMessage = reader.GetFieldValue(reader.GetOrdinal("is_system_message")), + ConversationId = reader.GetFieldValue(reader.GetOrdinal("conversation_id")), + Headers = [.. headers], + Status = (MessageStatus)reader.GetFieldValue(reader.GetOrdinal("status")), + MessageIntent = (MessageIntent)(messageMetadata.ContainsKey("MessageIntent") ? JsonSerializer.Deserialize((JsonElement)messageMetadata["MessageIntent"]) : 1), + BodyUrl = "", + BodySize = messageMetadata.ContainsKey("ContentLength") ? JsonSerializer.Deserialize((JsonElement)messageMetadata["ContentLength"]) : 0, + InvokedSagas = messageMetadata.ContainsKey("InvokedSagas") ? JsonSerializer.Deserialize>((JsonElement)messageMetadata["InvokedSagas"]) : [], + OriginatesFromSaga = messageMetadata.ContainsKey("OriginatesFromSaga") ? JsonSerializer.Deserialize((JsonElement)messageMetadata["OriginatesFromSaga"]) : null + }); + } + + return new QueryResult>(results, new QueryStatsInfo(string.Empty, results.Count)); + } } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs index 58a8237035..136f4ec0ba 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs @@ -7,7 +7,10 @@ class PostgreSQLConnectionFactory(DatabaseConfiguration databaseConfiguration) { public async Task OpenConnection(CancellationToken cancellationToken) { - var conn = new NpgsqlConnection(databaseConfiguration.ConnectionString); + var dataSourceBuilder = new NpgsqlDataSourceBuilder(databaseConfiguration.ConnectionString); + dataSourceBuilder.EnableDynamicJson(); + var dataSource = dataSourceBuilder.Build(); + var conn = dataSource.CreateConnection(); await conn.OpenAsync(cancellationToken); return conn; } @@ -18,7 +21,10 @@ public async Task OpenAdminConnection(CancellationToken cancel { Database = databaseConfiguration.AdminDatabaseName }; - var conn = new NpgsqlConnection(builder.ConnectionString); + var dataSourceBuilder = new NpgsqlDataSourceBuilder(builder.ConnectionString); + dataSourceBuilder.EnableDynamicJson(); + var dataSource = dataSourceBuilder.Build(); + var conn = dataSource.CreateConnection(); await conn.OpenAsync(cancellationToken); return conn; } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs index 8036c8c598..f13982e285 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs @@ -60,6 +60,7 @@ CREATE OR REPLACE FUNCTION processed_messages_tsvector_update() RETURNS trigger END $$ LANGUAGE plpgsql; + DROP TRIGGER IF EXISTS processed_messages_tsvector_trigger ON processed_messages; CREATE TRIGGER processed_messages_tsvector_trigger BEFORE INSERT OR UPDATE ON processed_messages FOR EACH ROW EXECUTE FUNCTION processed_messages_tsvector_update();", connection)) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj b/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj index 678bd2b03d..6c4264d0e0 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj @@ -4,6 +4,7 @@ net8.0 true true + enable From 7c2289cd181a8d63acb808365d5c167f28c2355f Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 28 Aug 2025 11:40:06 +1000 Subject: [PATCH 10/41] A bit better --- .../PostgreSQLAuditDataStore.cs | 74 ++++++++++++------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs index 4b74703d1c..391b0e2b76 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs @@ -15,8 +15,32 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; using ServiceControl.Audit.Persistence; using ServiceControl.SagaAudit; -class PostgreSQLAuditDataStore(PostgreSQLConnectionFactory connectionFactory) : IAuditDataStore +class PostgreSQLAuditDataStore : IAuditDataStore { + readonly PostgreSQLConnectionFactory connectionFactory; + public PostgreSQLAuditDataStore(PostgreSQLConnectionFactory connectionFactory) + { + this.connectionFactory = connectionFactory; + } + + // Helper to safely deserialize a value from a dictionary with a default + static T? DeserializeOrDefault(Dictionary dict, string key, T? defaultValue = default) + { + if (dict.TryGetValue(key, out var value) && value is JsonElement element && element.ValueKind != JsonValueKind.Null) + { + try + { + return JsonSerializer.Deserialize(element); + } + catch { } + } + return defaultValue; + } + + // Helper to get a value from the reader by column name + static T GetValue(NpgsqlDataReader reader, string column) + => reader.GetFieldValue(reader.GetOrdinal(column)); + public async Task GetMessageBody(string messageId, CancellationToken cancellationToken) { using var conn = await connectionFactory.OpenConnection(cancellationToken); @@ -41,7 +65,7 @@ public async Task GetMessageBody(string messageId, Cancellation } async Task>> GetAllMessages( - string? conversationId, bool? includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, + string? conversationId, bool? includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange, string? endpointName, string? q, CancellationToken cancellationToken) @@ -174,7 +198,7 @@ public async Task>> GetMessages(bool includeSyst public Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken) => throw new NotImplementedException(); public Task>> QueryKnownEndpoints(CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) + public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) { return GetAllMessages(null, null, pagingInfo, sortInfo, timeSentRange, searchParam, null, cancellationToken); } @@ -183,12 +207,12 @@ public async Task>> QueryMessagesByConversationI return await GetAllMessages(conversationId, null, pagingInfo, sortInfo, null, null, null, cancellationToken); } - public async Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) + public async Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) { return await GetAllMessages(null, includeSystemMessages, pagingInfo, sortInfo, timeSentRange, null, endpointName, cancellationToken); } - public Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) + public Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) { return GetAllMessages(null, null, pagingInfo, sortInfo, timeSentRange, keyword, endpoint, cancellationToken); } @@ -197,37 +221,35 @@ public Task>> QueryMessagesByReceivingEndpointAn async Task>> ReturnResults(NpgsqlCommand cmd, CancellationToken cancellationToken = default) { var results = new List(); - using var reader = await cmd.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { - var headers = reader.GetFieldValue>(reader.GetOrdinal("headers")); - var messageMetadata = reader.GetFieldValue>(reader.GetOrdinal("message_metadata")); + var headers = GetValue>(reader, "headers"); + var messageMetadata = GetValue>(reader, "message_metadata"); results.Add(new MessagesView { - Id = reader.GetFieldValue(reader.GetOrdinal("unique_message_id")), - MessageId = reader.GetFieldValue(reader.GetOrdinal("message_id")), - MessageType = reader.GetFieldValue(reader.GetOrdinal("message_type")), - SendingEndpoint = JsonSerializer.Deserialize((JsonElement)messageMetadata["SendingEndpoint"]), - ReceivingEndpoint = JsonSerializer.Deserialize((JsonElement)messageMetadata["ReceivingEndpoint"]), - TimeSent = reader.GetFieldValue(reader.GetOrdinal("time_sent")), - ProcessedAt = reader.GetFieldValue(reader.GetOrdinal("processed_at")), - CriticalTime = reader.GetFieldValue(reader.GetOrdinal("critical_time")), - ProcessingTime = reader.GetFieldValue(reader.GetOrdinal("processing_time")), - DeliveryTime = reader.GetFieldValue(reader.GetOrdinal("delivery_time")), - IsSystemMessage = reader.GetFieldValue(reader.GetOrdinal("is_system_message")), - ConversationId = reader.GetFieldValue(reader.GetOrdinal("conversation_id")), + Id = GetValue(reader, "unique_message_id"), + MessageId = GetValue(reader, "message_id"), + MessageType = GetValue(reader, "message_type"), + SendingEndpoint = DeserializeOrDefault(messageMetadata, "SendingEndpoint"), + ReceivingEndpoint = DeserializeOrDefault(messageMetadata, "ReceivingEndpoint"), + TimeSent = GetValue(reader, "time_sent"), + ProcessedAt = GetValue(reader, "processed_at"), + CriticalTime = GetValue(reader, "critical_time"), + ProcessingTime = GetValue(reader, "processing_time"), + DeliveryTime = GetValue(reader, "delivery_time"), + IsSystemMessage = GetValue(reader, "is_system_message"), + ConversationId = GetValue(reader, "conversation_id"), Headers = [.. headers], - Status = (MessageStatus)reader.GetFieldValue(reader.GetOrdinal("status")), - MessageIntent = (MessageIntent)(messageMetadata.ContainsKey("MessageIntent") ? JsonSerializer.Deserialize((JsonElement)messageMetadata["MessageIntent"]) : 1), + Status = (MessageStatus)GetValue(reader, "status"), + MessageIntent = (MessageIntent)DeserializeOrDefault(messageMetadata, "MessageIntent", 1), BodyUrl = "", - BodySize = messageMetadata.ContainsKey("ContentLength") ? JsonSerializer.Deserialize((JsonElement)messageMetadata["ContentLength"]) : 0, - InvokedSagas = messageMetadata.ContainsKey("InvokedSagas") ? JsonSerializer.Deserialize>((JsonElement)messageMetadata["InvokedSagas"]) : [], - OriginatesFromSaga = messageMetadata.ContainsKey("OriginatesFromSaga") ? JsonSerializer.Deserialize((JsonElement)messageMetadata["OriginatesFromSaga"]) : null + BodySize = DeserializeOrDefault(messageMetadata, "ContentLength", 0), + InvokedSagas = DeserializeOrDefault>(messageMetadata, "InvokedSagas", []), + OriginatesFromSaga = DeserializeOrDefault(messageMetadata, "OriginatesFromSaga") }); } - return new QueryResult>(results, new QueryStatsInfo(string.Empty, results.Count)); } } From 76e5c9ac7e9f86e6ca6cf445b8322a7a925be8d6 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 28 Aug 2025 11:59:59 +1000 Subject: [PATCH 11/41] Some refactoring --- .../PostgreSQLAuditDataStore.cs | 235 +++++------------- .../PostgresqlMessagesQueryBuilder.cs | 145 +++++++++++ 2 files changed, 214 insertions(+), 166 deletions(-) create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/PostgresqlMessagesQueryBuilder.cs diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs index 391b0e2b76..e4c3eaef27 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs @@ -1,8 +1,8 @@ + namespace ServiceControl.Audit.Persistence.PostgreSQL; using System; using System.Collections.Generic; -using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -15,209 +15,112 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; using ServiceControl.Audit.Persistence; using ServiceControl.SagaAudit; -class PostgreSQLAuditDataStore : IAuditDataStore -{ - readonly PostgreSQLConnectionFactory connectionFactory; - public PostgreSQLAuditDataStore(PostgreSQLConnectionFactory connectionFactory) - { - this.connectionFactory = connectionFactory; - } - - // Helper to safely deserialize a value from a dictionary with a default - static T? DeserializeOrDefault(Dictionary dict, string key, T? defaultValue = default) - { - if (dict.TryGetValue(key, out var value) && value is JsonElement element && element.ValueKind != JsonValueKind.Null) - { - try - { - return JsonSerializer.Deserialize(element); - } - catch { } - } - return defaultValue; - } - - // Helper to get a value from the reader by column name - static T GetValue(NpgsqlDataReader reader, string column) - => reader.GetFieldValue(reader.GetOrdinal(column)); +class PostgreSQLAuditDataStore(PostgreSQLConnectionFactory connectionFactory) : IAuditDataStore +{ public async Task GetMessageBody(string messageId, CancellationToken cancellationToken) { using var conn = await connectionFactory.OpenConnection(cancellationToken); - using var cmd = new NpgsqlCommand(@" select body, headers from processed_messages where message_id = @message_id LIMIT 1;", conn); - cmd.Parameters.AddWithValue("message_id", messageId); - using var reader = await cmd.ExecuteReaderAsync(cancellationToken); if (await reader.ReadAsync(cancellationToken)) { var stream = await reader.GetStreamAsync(reader.GetOrdinal("body"), cancellationToken); var contentType = reader.GetFieldValue>(reader.GetOrdinal("headers")).GetValueOrDefault(Headers.ContentType, "text/xml"); - return MessageBodyView.FromStream(stream, contentType, (int)stream.Length, string.Empty); } - return MessageBodyView.NotFound(); } - async Task>> GetAllMessages( - string? conversationId, bool? includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange, - string? endpointName, - string? q, - CancellationToken cancellationToken) - { - using var conn = await connectionFactory.OpenConnection(cancellationToken); - - var sql = new StringBuilder(@"select unique_message_id, - message_metadata, - headers, - processed_at, - message_id, - message_type, - is_system_message, - status, - time_sent, - receiving_endpoint_name, - critical_time, - processing_time, - delivery_time, - conversation_id from processed_messages - where 1 = 1"); - - if (includeSystemMessages.HasValue) - { - sql.Append(" and is_system_message = @is_system_message"); - } - - if (!string.IsNullOrWhiteSpace(q)) - { - sql.Append(" and query @@ to_tsquery('english', @search)"); - } - - if (!string.IsNullOrWhiteSpace(conversationId)) - { - sql.Append(" and conversation_id = @conversation_id"); - } - - if (!string.IsNullOrWhiteSpace(endpointName)) - { - sql.Append(" and receiving_endpoint_name = @endpoint_name"); - } - - if (timeSentRange?.From != null) - { - sql.Append(" and time_sent >= @time_sent_start"); - } - - if (timeSentRange?.To != null) - { - sql.Append(" and time_sent <= @time_sent_end"); - } - - sql.Append(" ORDER BY"); - switch (sortInfo.Sort) - { - - case "id": - case "message_id": - sql.Append(" message_id"); - break; - case "message_type": - sql.Append(" message_type"); - break; - case "critical_time": - sql.Append(" critical_time"); - break; - case "delivery_time": - sql.Append(" delivery_time"); - break; - case "processing_time": - sql.Append(" processing_time"); - break; - case "processed_at": - sql.Append(" processed_at"); - break; - case "status": - sql.Append(" status"); - break; - default: - sql.Append(" time_sent"); - break; - } - - if (sortInfo.Direction == "asc") - { - sql.Append(" ASC"); - } - else - { - sql.Append(" DESC"); - } - - sql.Append($" LIMIT {pagingInfo.PageSize} OFFSET {pagingInfo.Offset};"); - - var query = sql.ToString(); - using var cmd = new NpgsqlCommand(query, conn); - - if (!string.IsNullOrWhiteSpace(q)) - { - cmd.Parameters.AddWithValue("search", q); - } - if (!string.IsNullOrWhiteSpace(endpointName)) - { - cmd.Parameters.AddWithValue("endpoint_name", endpointName); - } - if (!string.IsNullOrWhiteSpace(conversationId)) - { - cmd.Parameters.AddWithValue("conversation_id", conversationId); - } - if (includeSystemMessages.HasValue) - { - cmd.Parameters.AddWithValue("is_system_message", includeSystemMessages); - } - if (timeSentRange?.From != null) - { - cmd.Parameters.AddWithValue("time_sent_start", timeSentRange.From); - } - if (timeSentRange?.To != null) - { - cmd.Parameters.AddWithValue("time_sent_end", timeSentRange.To); - } - - return await ReturnResults(cmd, cancellationToken); - } - - public async Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken) + public Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken) { - return await GetAllMessages(null, includeSystemMessages, pagingInfo, sortInfo, timeSentRange, null, null, cancellationToken); + var builder = new PostgresqlMessagesQueryBuilder() + .WithSystemMessages(includeSystemMessages) + .WithTimeSentRange(timeSentRange) + .WithSorting(sortInfo) + .WithPaging(pagingInfo); + return ExecuteMessagesQuery(builder, cancellationToken); } public Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken) => throw new NotImplementedException(); public Task>> QueryKnownEndpoints(CancellationToken cancellationToken) => throw new NotImplementedException(); public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) { - return GetAllMessages(null, null, pagingInfo, sortInfo, timeSentRange, searchParam, null, cancellationToken); + var builder = new PostgresqlMessagesQueryBuilder() + .WithSearch(searchParam) + .WithTimeSentRange(timeSentRange) + .WithSorting(sortInfo) + .WithPaging(pagingInfo); + return ExecuteMessagesQuery(builder, cancellationToken); } - public async Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) + + public Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) { - return await GetAllMessages(conversationId, null, pagingInfo, sortInfo, null, null, null, cancellationToken); + var builder = new PostgresqlMessagesQueryBuilder() + .WithConversationId(conversationId) + .WithSorting(sortInfo) + .WithPaging(pagingInfo); + return ExecuteMessagesQuery(builder, cancellationToken); } - public async Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) + public Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) { - return await GetAllMessages(null, includeSystemMessages, pagingInfo, sortInfo, timeSentRange, null, endpointName, cancellationToken); + var builder = new PostgresqlMessagesQueryBuilder() + .WithSystemMessages(includeSystemMessages) + .WithEndpointName(endpointName) + .WithTimeSentRange(timeSentRange) + .WithSorting(sortInfo) + .WithPaging(pagingInfo); + return ExecuteMessagesQuery(builder, cancellationToken); } public Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) { - return GetAllMessages(null, null, pagingInfo, sortInfo, timeSentRange, keyword, endpoint, cancellationToken); + var builder = new PostgresqlMessagesQueryBuilder() + .WithSearch(keyword) + .WithEndpointName(endpoint) + .WithTimeSentRange(timeSentRange) + .WithSorting(sortInfo) + .WithPaging(pagingInfo); + return ExecuteMessagesQuery(builder, cancellationToken); } + public Task> QuerySagaHistoryById(Guid input, CancellationToken cancellationToken) => throw new NotImplementedException(); + async Task>> ExecuteMessagesQuery( + PostgresqlMessagesQueryBuilder builder, + CancellationToken cancellationToken) + { + using var conn = await connectionFactory.OpenConnection(cancellationToken); + var (query, parameters) = builder.Build(); + using var cmd = new NpgsqlCommand(query, conn); + foreach (var param in parameters) + { + cmd.Parameters.Add(param); + } + return await ReturnResults(cmd, cancellationToken); + } + + static T? DeserializeOrDefault(Dictionary dict, string key, T? defaultValue = default) + { + if (dict.TryGetValue(key, out var value) && value is JsonElement element && element.ValueKind != JsonValueKind.Null) + { + try + { + return JsonSerializer.Deserialize(element); + } + catch { } + } + return defaultValue; + } + + static T GetValue(NpgsqlDataReader reader, string column) + => reader.GetFieldValue(reader.GetOrdinal(column)); + async Task>> ReturnResults(NpgsqlCommand cmd, CancellationToken cancellationToken = default) { var results = new List(); diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgresqlMessagesQueryBuilder.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgresqlMessagesQueryBuilder.cs new file mode 100644 index 0000000000..00323a558d --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgresqlMessagesQueryBuilder.cs @@ -0,0 +1,145 @@ + + + +namespace ServiceControl.Audit.Persistence.PostgreSQL; + +using System; +using System.Collections.Generic; +using System.Text; +using Npgsql; +using ServiceControl.Audit.Infrastructure; +public class PostgresqlMessagesQueryBuilder +{ + readonly StringBuilder sql = new(); + readonly List parameters = []; + + public PostgresqlMessagesQueryBuilder() + { + sql.Append(@"select unique_message_id, + message_metadata, + headers, + processed_at, + message_id, + message_type, + is_system_message, + status, + time_sent, + receiving_endpoint_name, + critical_time, + processing_time, + delivery_time, + conversation_id from processed_messages + where 1 = 1"); + } + + public PostgresqlMessagesQueryBuilder WithSystemMessages(bool? includeSystemMessages) + { + if (includeSystemMessages.HasValue) + { + sql.Append(" and is_system_message = @is_system_message"); + parameters.Add(new NpgsqlParameter("is_system_message", includeSystemMessages)); + } + return this; + } + + public PostgresqlMessagesQueryBuilder WithSearch(string? q) + { + if (!string.IsNullOrWhiteSpace(q)) + { + sql.Append(" and query @@ to_tsquery('english', @search)"); + parameters.Add(new NpgsqlParameter("search", q)); + } + return this; + } + + public PostgresqlMessagesQueryBuilder WithConversationId(string? conversationId) + { + if (!string.IsNullOrWhiteSpace(conversationId)) + { + sql.Append(" and conversation_id = @conversation_id"); + parameters.Add(new NpgsqlParameter("conversation_id", conversationId)); + } + return this; + } + + public PostgresqlMessagesQueryBuilder WithMessageId(string? messageId) + { + if (!string.IsNullOrWhiteSpace(messageId)) + { + sql.Append(" and message_id = @message_id"); + parameters.Add(new NpgsqlParameter("message_id", messageId)); + } + return this; + } + + public PostgresqlMessagesQueryBuilder WithEndpointName(string? endpointName) + { + if (!string.IsNullOrWhiteSpace(endpointName)) + { + sql.Append(" and receiving_endpoint_name = @endpoint_name"); + parameters.Add(new NpgsqlParameter("endpoint_name", endpointName)); + } + return this; + } + + public PostgresqlMessagesQueryBuilder WithTimeSentRange(DateTimeRange? timeSentRange) + { + if (timeSentRange?.From != null) + { + sql.Append(" and time_sent >= @time_sent_start"); + parameters.Add(new NpgsqlParameter("time_sent_start", timeSentRange.From)); + } + if (timeSentRange?.To != null) + { + sql.Append(" and time_sent <= @time_sent_end"); + parameters.Add(new NpgsqlParameter("time_sent_end", timeSentRange.To)); + } + return this; + } + + public PostgresqlMessagesQueryBuilder WithSorting(SortInfo sortInfo) + { + sql.Append(" ORDER BY"); + switch (sortInfo.Sort) + { + case "id": + case "message_id": + sql.Append(" message_id"); + break; + case "message_type": + sql.Append(" message_type"); + break; + case "critical_time": + sql.Append(" critical_time"); + break; + case "delivery_time": + sql.Append(" delivery_time"); + break; + case "processing_time": + sql.Append(" processing_time"); + break; + case "processed_at": + sql.Append(" processed_at"); + break; + case "status": + sql.Append(" status"); + break; + default: + sql.Append(" time_sent"); + break; + } + sql.Append(sortInfo.Direction == "asc" ? " ASC" : " DESC"); + return this; + } + + public PostgresqlMessagesQueryBuilder WithPaging(PagingInfo pagingInfo) + { + sql.Append($" LIMIT {pagingInfo.PageSize} OFFSET {pagingInfo.Offset};"); + return this; + } + + public (string Sql, List Parameters) Build() + { + return (sql.ToString(), parameters); + } +} From 176fba3298d5622de6a0aa3e28a189fd27320e76 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 28 Aug 2025 13:04:04 +1000 Subject: [PATCH 12/41] Adding prepare for the inserts --- .../UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index 3c1c5b2743..b2e6ac3d20 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -70,6 +70,7 @@ INSERT INTO processed_messages ( cmd.Parameters.AddWithValue("conversation_id", GetMetadata("ConversationId")); cmd.Parameters.AddWithValue("status", (int)(GetMetadata("IsRetried") ? MessageStatus.ResolvedSuccessfully : MessageStatus.Successful)); + await cmd.PrepareAsync(cancellationToken); await cmd.ExecuteNonQueryAsync(cancellationToken); } @@ -96,6 +97,7 @@ INSERT INTO saga_snapshots ( cmd.Parameters.AddWithValue("endpoint", sagaSnapshot.Endpoint ?? (object)DBNull.Value); cmd.Parameters.AddWithValue("processed_at", sagaSnapshot.ProcessedAt); + await cmd.PrepareAsync(cancellationToken); await cmd.ExecuteNonQueryAsync(cancellationToken); } @@ -116,6 +118,7 @@ INSERT INTO known_endpoints ( cmd.Parameters.AddWithValue("host", knownEndpoint.Host ?? (object)DBNull.Value); cmd.Parameters.AddWithValue("last_seen", knownEndpoint.LastSeen); + await cmd.PrepareAsync(cancellationToken); await cmd.ExecuteNonQueryAsync(cancellationToken); } } \ No newline at end of file From 04c887f8533a6f0c68796677fb96c64697040ac5 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 28 Aug 2025 13:20:58 +1000 Subject: [PATCH 13/41] Adding batch --- .../PostgreSQLAuditIngestionUnitOfWork.cs | 45 +++++++++---------- ...stgreSQLAuditIngestionUnitOfWorkFactory.cs | 3 +- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index b2e6ac3d20..0fd4c36d2c 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -13,28 +13,27 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; class PostgreSQLAuditIngestionUnitOfWork : IAuditIngestionUnitOfWork { - readonly NpgsqlConnection connection; - readonly NpgsqlTransaction transaction; + readonly NpgsqlBatch batch; - public PostgreSQLAuditIngestionUnitOfWork(NpgsqlConnection connection, NpgsqlTransaction transaction) + public PostgreSQLAuditIngestionUnitOfWork(NpgsqlConnection connection) { - this.connection = connection; - this.transaction = transaction; + batch = new NpgsqlBatch(connection); } public async ValueTask DisposeAsync() { - await transaction.CommitAsync(); - await transaction.DisposeAsync(); - await connection.DisposeAsync(); + await batch.PrepareAsync(); + await batch.ExecuteNonQueryAsync(); + await batch.DisposeAsync(); } - public async Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body, CancellationToken cancellationToken) + public Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body, CancellationToken cancellationToken) { T GetMetadata(string key) => processedMessage.MessageMetadata.TryGetValue(key, out var value) ? (T)value ?? default : default; // Insert ProcessedMessage into processed_messages table - var cmd = new NpgsqlCommand(@" + var cmd = batch.CreateBatchCommand(); + cmd.CommandText = @" INSERT INTO processed_messages ( unique_message_id, message_metadata, headers, processed_at, body, message_id, message_type, is_system_message, status, time_sent, receiving_endpoint_name, @@ -43,8 +42,7 @@ INSERT INTO processed_messages ( @unique_message_id, @message_metadata, @headers, @processed_at, @body, @message_id, @message_type, @is_system_message, @status, @time_sent, @receiving_endpoint_name, @critical_time, @processing_time, @delivery_time, @conversation_id - ) - ;", connection, transaction); + );"; processedMessage.MessageMetadata["ContentLength"] = body.Length; if (!body.IsEmpty) @@ -70,20 +68,20 @@ INSERT INTO processed_messages ( cmd.Parameters.AddWithValue("conversation_id", GetMetadata("ConversationId")); cmd.Parameters.AddWithValue("status", (int)(GetMetadata("IsRetried") ? MessageStatus.ResolvedSuccessfully : MessageStatus.Successful)); - await cmd.PrepareAsync(cancellationToken); - await cmd.ExecuteNonQueryAsync(cancellationToken); + return Task.CompletedTask; } - public async Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken) + public Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken) { // Insert SagaSnapshot into saga_snapshots table - var cmd = new NpgsqlCommand(@" + var cmd = batch.CreateBatchCommand(); + cmd.CommandText = @" INSERT INTO saga_snapshots ( id, saga_id, saga_type, start_time, finish_time, status, state_after_change, initiating_message, outgoing_messages, endpoint, processed_at ) VALUES ( @id, @saga_id, @saga_type, @start_time, @finish_time, @status, @state_after_change, @initiating_message, @outgoing_messages, @endpoint, @processed_at ) - ON CONFLICT (id) DO NOTHING;", connection, transaction); + ON CONFLICT (id) DO NOTHING;"; cmd.Parameters.AddWithValue("id", sagaSnapshot.Id ?? (object)DBNull.Value); cmd.Parameters.AddWithValue("saga_id", sagaSnapshot.SagaId); @@ -97,20 +95,20 @@ INSERT INTO saga_snapshots ( cmd.Parameters.AddWithValue("endpoint", sagaSnapshot.Endpoint ?? (object)DBNull.Value); cmd.Parameters.AddWithValue("processed_at", sagaSnapshot.ProcessedAt); - await cmd.PrepareAsync(cancellationToken); - await cmd.ExecuteNonQueryAsync(cancellationToken); + return Task.CompletedTask; } - public async Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken) + public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken) { // Insert KnownEndpoint into known_endpoints table - var cmd = new NpgsqlCommand(@" + var cmd = batch.CreateBatchCommand(); + cmd.CommandText = @" INSERT INTO known_endpoints ( id, name, host_id, host, last_seen ) VALUES ( @id, @name, @host_id, @host, @last_seen ) - ON CONFLICT (id) DO NOTHING;", connection, transaction); + ON CONFLICT (id) DO NOTHING;"; cmd.Parameters.AddWithValue("id", knownEndpoint.Id ?? (object)DBNull.Value); cmd.Parameters.AddWithValue("name", knownEndpoint.Name ?? (object)DBNull.Value); @@ -118,7 +116,6 @@ INSERT INTO known_endpoints ( cmd.Parameters.AddWithValue("host", knownEndpoint.Host ?? (object)DBNull.Value); cmd.Parameters.AddWithValue("last_seen", knownEndpoint.LastSeen); - await cmd.PrepareAsync(cancellationToken); - await cmd.ExecuteNonQueryAsync(cancellationToken); + return Task.CompletedTask; } } \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs index 25649fb48c..d51d835c00 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs @@ -16,8 +16,7 @@ public PostgreSQLAuditIngestionUnitOfWorkFactory(PostgreSQLConnectionFactory con public async ValueTask StartNew(int batchSize, CancellationToken cancellationToken) { var connection = await connectionFactory.OpenConnection(cancellationToken); - var transaction = await connection.BeginTransactionAsync(cancellationToken); - return new PostgreSQLAuditIngestionUnitOfWork(connection, transaction); + return new PostgreSQLAuditIngestionUnitOfWork(connection); } public bool CanIngestMore() => true; From 91d47e5fbfce0d81a8f170d568b1cc531cb627e2 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 28 Aug 2025 13:36:58 +1000 Subject: [PATCH 14/41] remove explicit serialization to json --- .../PostgreSQLAuditIngestionUnitOfWork.cs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index 0fd4c36d2c..daa5b19202 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -53,9 +53,9 @@ INSERT INTO processed_messages ( { cmd.Parameters.AddWithValue("body", DBNull.Value); } - cmd.Parameters.AddWithValue("unique_message_id", processedMessage.UniqueMessageId ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("message_metadata", JsonSerializer.SerializeToDocument(processedMessage.MessageMetadata)); - cmd.Parameters.AddWithValue("headers", JsonSerializer.SerializeToDocument(processedMessage.Headers)); + cmd.Parameters.AddWithValue("unique_message_id", processedMessage.UniqueMessageId); + cmd.Parameters.AddWithValue("message_metadata", NpgsqlTypes.NpgsqlDbType.Jsonb, processedMessage.MessageMetadata); + cmd.Parameters.AddWithValue("headers", NpgsqlTypes.NpgsqlDbType.Jsonb, processedMessage.Headers); cmd.Parameters.AddWithValue("processed_at", processedMessage.ProcessedAt); cmd.Parameters.AddWithValue("message_id", GetMetadata("MessageId")); cmd.Parameters.AddWithValue("message_type", GetMetadata("MessageType")); @@ -83,16 +83,16 @@ INSERT INTO saga_snapshots ( ) ON CONFLICT (id) DO NOTHING;"; - cmd.Parameters.AddWithValue("id", sagaSnapshot.Id ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("id", sagaSnapshot.Id); cmd.Parameters.AddWithValue("saga_id", sagaSnapshot.SagaId); - cmd.Parameters.AddWithValue("saga_type", sagaSnapshot.SagaType ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("saga_type", sagaSnapshot.SagaType); cmd.Parameters.AddWithValue("start_time", sagaSnapshot.StartTime); cmd.Parameters.AddWithValue("finish_time", sagaSnapshot.FinishTime); cmd.Parameters.AddWithValue("status", sagaSnapshot.Status.ToString()); - cmd.Parameters.AddWithValue("state_after_change", sagaSnapshot.StateAfterChange ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("initiating_message", JsonSerializer.Serialize(sagaSnapshot.InitiatingMessage)); - cmd.Parameters.AddWithValue("outgoing_messages", JsonSerializer.Serialize(sagaSnapshot.OutgoingMessages)); - cmd.Parameters.AddWithValue("endpoint", sagaSnapshot.Endpoint ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("state_after_change", sagaSnapshot.StateAfterChange); + cmd.Parameters.AddWithValue("initiating_message", NpgsqlTypes.NpgsqlDbType.Jsonb, sagaSnapshot.InitiatingMessage); + cmd.Parameters.AddWithValue("outgoing_messages", NpgsqlTypes.NpgsqlDbType.Jsonb, sagaSnapshot.OutgoingMessages); + cmd.Parameters.AddWithValue("endpoint", sagaSnapshot.Endpoint); cmd.Parameters.AddWithValue("processed_at", sagaSnapshot.ProcessedAt); return Task.CompletedTask; @@ -110,10 +110,10 @@ INSERT INTO known_endpoints ( ) ON CONFLICT (id) DO NOTHING;"; - cmd.Parameters.AddWithValue("id", knownEndpoint.Id ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("name", knownEndpoint.Name ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("id", knownEndpoint.Id); + cmd.Parameters.AddWithValue("name", knownEndpoint.Name); cmd.Parameters.AddWithValue("host_id", knownEndpoint.HostId); - cmd.Parameters.AddWithValue("host", knownEndpoint.Host ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("host", knownEndpoint.Host); cmd.Parameters.AddWithValue("last_seen", knownEndpoint.LastSeen); return Task.CompletedTask; From 8c7be1a9b3b0ecbca2489af70d7fbc548c090426 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 28 Aug 2025 18:00:42 +1000 Subject: [PATCH 15/41] Fixed a few bugs --- .../PostgresqlMessagesQueryBuilder.cs | 2 +- .../UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgresqlMessagesQueryBuilder.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgresqlMessagesQueryBuilder.cs index 00323a558d..61aa98b22d 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgresqlMessagesQueryBuilder.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgresqlMessagesQueryBuilder.cs @@ -46,7 +46,7 @@ public PostgresqlMessagesQueryBuilder WithSearch(string? q) { if (!string.IsNullOrWhiteSpace(q)) { - sql.Append(" and query @@ to_tsquery('english', @search)"); + sql.Append(" and query @@ plainto_tsquery('english', @search)"); parameters.Add(new NpgsqlParameter("search", q)); } return this; diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index daa5b19202..3ece5efe12 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -61,13 +61,14 @@ INSERT INTO processed_messages ( cmd.Parameters.AddWithValue("message_type", GetMetadata("MessageType")); cmd.Parameters.AddWithValue("is_system_message", GetMetadata("IsSystemMessage")); cmd.Parameters.AddWithValue("time_sent", GetMetadata("TimeSent")); - cmd.Parameters.AddWithValue("receiving_endpoint_name", GetMetadata("ReceivingEndpoint").Name); + cmd.Parameters.AddWithValue("receiving_endpoint_name", GetMetadata("ReceivingEndpoint")?.Name ?? (object)DBNull.Value); cmd.Parameters.AddWithValue("critical_time", GetMetadata("CriticalTime")); cmd.Parameters.AddWithValue("processing_time", GetMetadata("ProcessingTime")); cmd.Parameters.AddWithValue("delivery_time", GetMetadata("DeliveryTime")); cmd.Parameters.AddWithValue("conversation_id", GetMetadata("ConversationId")); cmd.Parameters.AddWithValue("status", (int)(GetMetadata("IsRetried") ? MessageStatus.ResolvedSuccessfully : MessageStatus.Successful)); + batch.BatchCommands.Add(cmd); return Task.CompletedTask; } @@ -94,6 +95,7 @@ INSERT INTO saga_snapshots ( cmd.Parameters.AddWithValue("outgoing_messages", NpgsqlTypes.NpgsqlDbType.Jsonb, sagaSnapshot.OutgoingMessages); cmd.Parameters.AddWithValue("endpoint", sagaSnapshot.Endpoint); cmd.Parameters.AddWithValue("processed_at", sagaSnapshot.ProcessedAt); + batch.BatchCommands.Add(cmd); return Task.CompletedTask; } @@ -115,6 +117,7 @@ INSERT INTO known_endpoints ( cmd.Parameters.AddWithValue("host_id", knownEndpoint.HostId); cmd.Parameters.AddWithValue("host", knownEndpoint.Host); cmd.Parameters.AddWithValue("last_seen", knownEndpoint.LastSeen); + batch.BatchCommands.Add(cmd); return Task.CompletedTask; } From 48c2fe49f94810eba8972ca5b1235b89a1079b0d Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 28 Aug 2025 18:00:59 +1000 Subject: [PATCH 16/41] Implemented more methods --- .../PostgreSQLAuditDataStore.cs | 72 ++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs index e4c3eaef27..11f2dc591d 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs @@ -13,6 +13,7 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; using ServiceControl.Audit.Infrastructure; using ServiceControl.Audit.Monitoring; using ServiceControl.Audit.Persistence; +using ServiceControl.Audit.Persistence.Infrastructure; using ServiceControl.SagaAudit; @@ -46,8 +47,75 @@ public Task>> GetMessages(bool includeSystemMess return ExecuteMessagesQuery(builder, cancellationToken); } - public Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task>> QueryKnownEndpoints(CancellationToken cancellationToken) => throw new NotImplementedException(); + public async Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken) + { + var startDate = DateTime.UtcNow.AddDays(-30); + var endDate = DateTime.UtcNow; + using var connection = await connectionFactory.OpenConnection(cancellationToken); + using var cmd = new NpgsqlCommand(@" + SELECT + DATE_TRUNC('day', processed_at) AS day, + COUNT(*) AS count + FROM processed_messages + WHERE receiving_endpoint_name = @endpoint_name + AND processed_at BETWEEN @start_date AND @end_date + GROUP BY day + ORDER BY day;", connection); + cmd.Parameters.AddWithValue("endpoint_name", endpointName); + cmd.Parameters.AddWithValue("start_date", startDate); + cmd.Parameters.AddWithValue("end_date", endDate); + + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + var results = new List(); + while (await reader.ReadAsync(cancellationToken)) + { + results.Add(new AuditCount + { + UtcDate = reader.GetDateTime(reader.GetOrdinal("day")), + Count = reader.GetInt32(reader.GetOrdinal("count")) + }); + } + + return new QueryResult>(results, new QueryStatsInfo(string.Empty, results.Count)); + } + + public async Task>> QueryKnownEndpoints(CancellationToken cancellationToken) + { + // We need to return all the data from known_endpoints table in postgress + using var connection = await connectionFactory.OpenConnection(cancellationToken); + using var cmd = new NpgsqlCommand(@" + SELECT + id, + name, + host_id, + host, + last_seen + FROM known_endpoints;", connection); + + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + var results = new List(); + while (await reader.ReadAsync(cancellationToken)) + { + var name = reader.GetString(reader.GetOrdinal("name")); + var hostId = reader.GetGuid(reader.GetOrdinal("host_id")); + var host = reader.GetString(reader.GetOrdinal("host")); + var lastSeen = reader.GetDateTime(reader.GetOrdinal("last_seen")); + results.Add(new KnownEndpointsView + { + Id = DeterministicGuid.MakeId(name, hostId.ToString()), + EndpointDetails = new EndpointDetails + { + Host = host, + HostId = hostId, + Name = name + }, + HostDisplayName = host + }); + } + + return new QueryResult>(results, new QueryStatsInfo(string.Empty, results.Count)); + } + public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) { var builder = new PostgresqlMessagesQueryBuilder() From 5642bf3d5908c02f00c42d350c2d4f83ab2e6d27 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 28 Aug 2025 18:34:58 +1000 Subject: [PATCH 17/41] Add url for message body --- .../PostgreSQLAuditDataStore.cs | 6 ++++-- .../UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs index 11f2dc591d..769a530eec 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs @@ -30,7 +30,8 @@ public async Task GetMessageBody(string messageId, Cancellation using var reader = await cmd.ExecuteReaderAsync(cancellationToken); if (await reader.ReadAsync(cancellationToken)) { - var stream = await reader.GetStreamAsync(reader.GetOrdinal("body"), cancellationToken); + //var stream = await reader.GetStreamAsync(reader.GetOrdinal("body"), cancellationToken); + var stream = reader.GetStream(reader.GetOrdinal("body")); var contentType = reader.GetFieldValue>(reader.GetOrdinal("headers")).GetValueOrDefault(Headers.ContentType, "text/xml"); return MessageBodyView.FromStream(stream, contentType, (int)stream.Length, string.Empty); } @@ -215,7 +216,7 @@ async Task>> ReturnResults(NpgsqlCommand cmd, Ca Headers = [.. headers], Status = (MessageStatus)GetValue(reader, "status"), MessageIntent = (MessageIntent)DeserializeOrDefault(messageMetadata, "MessageIntent", 1), - BodyUrl = "", + BodyUrl = string.Format(BodyUrlFormatString, GetValue(reader, "message_id")), BodySize = DeserializeOrDefault(messageMetadata, "ContentLength", 0), InvokedSagas = DeserializeOrDefault>(messageMetadata, "InvokedSagas", []), OriginatesFromSaga = DeserializeOrDefault(messageMetadata, "OriginatesFromSaga") @@ -223,4 +224,5 @@ async Task>> ReturnResults(NpgsqlCommand cmd, Ca } return new QueryResult>(results, new QueryStatsInfo(string.Empty, results.Count)); } + public const string BodyUrlFormatString = "/messages/{0}/body"; } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index 3ece5efe12..ccaa2825e4 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -1,7 +1,6 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; using System; -using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Npgsql; From 2e968799341ae7ced73f70034fe96bb24cc560a0 Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 29 Aug 2025 08:20:12 +1000 Subject: [PATCH 18/41] Fix issue with body retrieval api --- src/Directory.Packages.props | 1 + .../PostgreSQLAuditDataStore.cs | 37 +++++++++++-------- .../PostgreSQLPersistenceInstaller.cs | 16 ++++---- ...ontrol.Audit.Persistence.PostgreSQL.csproj | 1 + 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index f8b8956d52..b5ff262baa 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -28,6 +28,7 @@ + diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs index 769a530eec..c8e2ae2047 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs @@ -6,6 +6,7 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using Microsoft.IO; using Npgsql; using NServiceBus; using ServiceControl.Audit.Auditing; @@ -19,21 +20,25 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; class PostgreSQLAuditDataStore(PostgreSQLConnectionFactory connectionFactory) : IAuditDataStore { + static readonly RecyclableMemoryStreamManager manager = new RecyclableMemoryStreamManager(); + public async Task GetMessageBody(string messageId, CancellationToken cancellationToken) { - using var conn = await connectionFactory.OpenConnection(cancellationToken); - using var cmd = new NpgsqlCommand(@" - select body, headers from processed_messages + await using var conn = await connectionFactory.OpenConnection(cancellationToken); + await using var cmd = new NpgsqlCommand(@" + select headers, body from processed_messages where message_id = @message_id LIMIT 1;", conn); cmd.Parameters.AddWithValue("message_id", messageId); - using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + await using var reader = await cmd.ExecuteReaderAsync(System.Data.CommandBehavior.SequentialAccess, cancellationToken); if (await reader.ReadAsync(cancellationToken)) { - //var stream = await reader.GetStreamAsync(reader.GetOrdinal("body"), cancellationToken); - var stream = reader.GetStream(reader.GetOrdinal("body")); var contentType = reader.GetFieldValue>(reader.GetOrdinal("headers")).GetValueOrDefault(Headers.ContentType, "text/xml"); - return MessageBodyView.FromStream(stream, contentType, (int)stream.Length, string.Empty); + using var stream = await reader.GetStreamAsync(reader.GetOrdinal("body"), cancellationToken); + var responseStream = manager.GetStream(); + await stream.CopyToAsync(responseStream, cancellationToken); + responseStream.Position = 0; + return MessageBodyView.FromStream(responseStream, contentType, (int)stream.Length, string.Empty); } return MessageBodyView.NotFound(); } @@ -52,8 +57,8 @@ public async Task>> QueryAuditCounts(string endpoi { var startDate = DateTime.UtcNow.AddDays(-30); var endDate = DateTime.UtcNow; - using var connection = await connectionFactory.OpenConnection(cancellationToken); - using var cmd = new NpgsqlCommand(@" + await using var connection = await connectionFactory.OpenConnection(cancellationToken); + await using var cmd = new NpgsqlCommand(@" SELECT DATE_TRUNC('day', processed_at) AS day, COUNT(*) AS count @@ -66,7 +71,7 @@ GROUP BY day cmd.Parameters.AddWithValue("start_date", startDate); cmd.Parameters.AddWithValue("end_date", endDate); - using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); var results = new List(); while (await reader.ReadAsync(cancellationToken)) { @@ -83,8 +88,8 @@ GROUP BY day public async Task>> QueryKnownEndpoints(CancellationToken cancellationToken) { // We need to return all the data from known_endpoints table in postgress - using var connection = await connectionFactory.OpenConnection(cancellationToken); - using var cmd = new NpgsqlCommand(@" + await using var connection = await connectionFactory.OpenConnection(cancellationToken); + await using var cmd = new NpgsqlCommand(@" SELECT id, name, @@ -93,7 +98,7 @@ public async Task>> QueryKnownEndpoints(Ca last_seen FROM known_endpoints;", connection); - using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); var results = new List(); while (await reader.ReadAsync(cancellationToken)) { @@ -164,9 +169,9 @@ async Task>> ExecuteMessagesQuery( PostgresqlMessagesQueryBuilder builder, CancellationToken cancellationToken) { - using var conn = await connectionFactory.OpenConnection(cancellationToken); + await using var conn = await connectionFactory.OpenConnection(cancellationToken); var (query, parameters) = builder.Build(); - using var cmd = new NpgsqlCommand(query, conn); + await using var cmd = new NpgsqlCommand(query, conn); foreach (var param in parameters) { cmd.Parameters.Add(param); @@ -193,7 +198,7 @@ static T GetValue(NpgsqlDataReader reader, string column) async Task>> ReturnResults(NpgsqlCommand cmd, CancellationToken cancellationToken = default) { var results = new List(); - using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { var headers = GetValue>(reader, "headers"); diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs index f13982e285..0aee1e2edf 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs @@ -10,9 +10,9 @@ class PostgreSQLPersistenceInstaller(DatabaseConfiguration databaseConfiguration { public async Task StartAsync(CancellationToken cancellationToken) { - using var adminConnection = await connectionFactory.OpenAdminConnection(cancellationToken); + await using var adminConnection = await connectionFactory.OpenAdminConnection(cancellationToken); - using (var cmd = new NpgsqlCommand($"SELECT 1 FROM pg_database WHERE datname = @dbname", adminConnection)) + await using (var cmd = new NpgsqlCommand($"SELECT 1 FROM pg_database WHERE datname = @dbname", adminConnection)) { cmd.Parameters.AddWithValue("@dbname", databaseConfiguration.Name); var exists = await cmd.ExecuteScalarAsync(cancellationToken); @@ -23,9 +23,9 @@ public async Task StartAsync(CancellationToken cancellationToken) } } - using var connection = await connectionFactory.OpenConnection(cancellationToken); + await using var connection = await connectionFactory.OpenConnection(cancellationToken); // Create processed_messages table - using (var cmd = new NpgsqlCommand(@" + await using (var cmd = new NpgsqlCommand(@" CREATE TABLE IF NOT EXISTS processed_messages ( id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, unique_message_id TEXT, @@ -50,7 +50,7 @@ query tsvector } // Create trigger for full text search - using (var cmd = new NpgsqlCommand(@" + await using (var cmd = new NpgsqlCommand(@" CREATE OR REPLACE FUNCTION processed_messages_tsvector_update() RETURNS trigger AS $$ BEGIN NEW.query := @@ -68,7 +68,7 @@ BEFORE INSERT OR UPDATE ON processed_messages await cmd.ExecuteNonQueryAsync(cancellationToken); } // Create index on processed_messages for specified columns - using (var cmd = new NpgsqlCommand(@" + await using (var cmd = new NpgsqlCommand(@" CREATE INDEX IF NOT EXISTS idx_processed_messages_multi ON processed_messages ( message_id, time_sent, @@ -84,7 +84,7 @@ CREATE INDEX IF NOT EXISTS idx_processed_messages_multi ON processed_messages ( } // Create saga_snapshots table - using (var cmd = new NpgsqlCommand(@" + await using (var cmd = new NpgsqlCommand(@" CREATE TABLE IF NOT EXISTS saga_snapshots ( id TEXT PRIMARY KEY, saga_id UUID, @@ -103,7 +103,7 @@ processed_at TIMESTAMPTZ } // Create known_endpoints table - using (var cmd = new NpgsqlCommand(@" + await using (var cmd = new NpgsqlCommand(@" CREATE TABLE IF NOT EXISTS known_endpoints ( id TEXT PRIMARY KEY, name TEXT, diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj b/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj index 6c4264d0e0..96ad791bdf 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj @@ -23,6 +23,7 @@ + From 9fc490e1b6f61d115da0ec67580628bfac6f3858 Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 29 Aug 2025 13:26:43 +1000 Subject: [PATCH 19/41] Fixed issue with pool of connections --- .../PostgreSQLConnectionFactory.cs | 45 +++++++++++++------ .../PostgreSQLAuditIngestionUnitOfWork.cs | 3 ++ ...stgreSQLAuditIngestionUnitOfWorkFactory.cs | 1 + 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs index 136f4ec0ba..2ee56dc253 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs @@ -3,28 +3,45 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; using Npgsql; using System.Threading.Tasks; using System.Threading; -class PostgreSQLConnectionFactory(DatabaseConfiguration databaseConfiguration) +using Microsoft.Extensions.Logging; + +class PostgreSQLConnectionFactory { - public async Task OpenConnection(CancellationToken cancellationToken) + readonly NpgsqlDataSource dataSource; + readonly NpgsqlDataSource dataSourceAdmin; + + public PostgreSQLConnectionFactory(DatabaseConfiguration databaseConfiguration, ILoggerFactory loggerFactory) { - var dataSourceBuilder = new NpgsqlDataSourceBuilder(databaseConfiguration.ConnectionString); + var dataSourceBuilder = new NpgsqlDataSourceBuilder(databaseConfiguration.ConnectionString) + { + Name = "ServiceControl.Audit" + }; + dataSourceBuilder.UseLoggerFactory(loggerFactory); dataSourceBuilder.EnableDynamicJson(); - var dataSource = dataSourceBuilder.Build(); - var conn = dataSource.CreateConnection(); - await conn.OpenAsync(cancellationToken); - return conn; - } + dataSource = dataSourceBuilder.Build(); - public async Task OpenAdminConnection(CancellationToken cancellationToken) - { var builder = new NpgsqlConnectionStringBuilder(databaseConfiguration.ConnectionString) { Database = databaseConfiguration.AdminDatabaseName }; - var dataSourceBuilder = new NpgsqlDataSourceBuilder(builder.ConnectionString); - dataSourceBuilder.EnableDynamicJson(); - var dataSource = dataSourceBuilder.Build(); - var conn = dataSource.CreateConnection(); + var dataSourceBuilderAdmin = new NpgsqlDataSourceBuilder(builder.ConnectionString) + { + Name = "ServiceControl.Audit-admin", + }; + dataSourceBuilderAdmin.UseLoggerFactory(loggerFactory); + dataSourceBuilderAdmin.EnableDynamicJson(); + dataSourceAdmin = dataSourceBuilderAdmin.Build(); + } + + public async Task OpenConnection(CancellationToken cancellationToken) + { + var conn = await dataSource.OpenConnectionAsync(cancellationToken); + return conn; + } + + public async Task OpenAdminConnection(CancellationToken cancellationToken) + { + var conn = dataSourceAdmin.CreateConnection(); await conn.OpenAsync(cancellationToken); return conn; } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index ccaa2825e4..9e45db6b7b 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -13,10 +13,12 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; class PostgreSQLAuditIngestionUnitOfWork : IAuditIngestionUnitOfWork { readonly NpgsqlBatch batch; + readonly NpgsqlConnection connection; public PostgreSQLAuditIngestionUnitOfWork(NpgsqlConnection connection) { batch = new NpgsqlBatch(connection); + this.connection = connection; } public async ValueTask DisposeAsync() @@ -24,6 +26,7 @@ public async ValueTask DisposeAsync() await batch.PrepareAsync(); await batch.ExecuteNonQueryAsync(); await batch.DisposeAsync(); + await connection.DisposeAsync(); } public Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body, CancellationToken cancellationToken) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs index d51d835c00..e5d805cebb 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs @@ -4,6 +4,7 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; using System.Threading.Tasks; using ServiceControl.Audit.Persistence.UnitOfWork; using ServiceControl.Audit.Persistence.PostgreSQL; + class PostgreSQLAuditIngestionUnitOfWorkFactory : IAuditIngestionUnitOfWorkFactory { readonly PostgreSQLConnectionFactory connectionFactory; From 772bbbdfeace554e8f070ab3895b49acf1ff4035 Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 29 Aug 2025 15:58:53 +1000 Subject: [PATCH 20/41] Remove logging, it is a bit noisy --- .../PostgreSQLConnectionFactory.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs index 2ee56dc253..00df7cd323 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs @@ -16,7 +16,7 @@ public PostgreSQLConnectionFactory(DatabaseConfiguration databaseConfiguration, { Name = "ServiceControl.Audit" }; - dataSourceBuilder.UseLoggerFactory(loggerFactory); + //dataSourceBuilder.UseLoggerFactory(loggerFactory); dataSourceBuilder.EnableDynamicJson(); dataSource = dataSourceBuilder.Build(); @@ -28,7 +28,7 @@ public PostgreSQLConnectionFactory(DatabaseConfiguration databaseConfiguration, { Name = "ServiceControl.Audit-admin", }; - dataSourceBuilderAdmin.UseLoggerFactory(loggerFactory); + //dataSourceBuilderAdmin.UseLoggerFactory(loggerFactory); dataSourceBuilderAdmin.EnableDynamicJson(); dataSourceAdmin = dataSourceBuilderAdmin.Build(); } From 2d0068474d53cd348c6f52f8a34e2f5ba6092115 Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 29 Aug 2025 15:59:36 +1000 Subject: [PATCH 21/41] Adding more granular indexes and adjusting autovacuum settings --- .../PostgreSQLPersistenceInstaller.cs | 123 ++++++++++++++---- 1 file changed, 97 insertions(+), 26 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs index 0aee1e2edf..e92f6e8df5 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs @@ -27,24 +27,29 @@ public async Task StartAsync(CancellationToken cancellationToken) // Create processed_messages table await using (var cmd = new NpgsqlCommand(@" CREATE TABLE IF NOT EXISTS processed_messages ( - id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - unique_message_id TEXT, - message_metadata JSONB, - headers JSONB, - processed_at TIMESTAMPTZ, - body BYTEA, - message_id TEXT, - message_type TEXT, - is_system_message BOOLEAN, - status NUMERIC, - time_sent TIMESTAMPTZ, - receiving_endpoint_name TEXT, - critical_time INTERVAL, - processing_time INTERVAL, - delivery_time INTERVAL, - conversation_id TEXT, - query tsvector - );", connection)) + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + unique_message_id TEXT, + message_metadata JSONB, + headers JSONB, + processed_at TIMESTAMPTZ, + body BYTEA, + message_id TEXT, + message_type TEXT, + is_system_message BOOLEAN, + status NUMERIC, + time_sent TIMESTAMPTZ, + receiving_endpoint_name TEXT, + critical_time INTERVAL, + processing_time INTERVAL, + delivery_time INTERVAL, + conversation_id TEXT, + query tsvector, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + WITH ( + autovacuum_vacuum_scale_factor = 0.05, + autovacuum_analyze_scale_factor = 0.02 + );", connection)) { await cmd.ExecuteNonQueryAsync(cancellationToken); } @@ -67,22 +72,88 @@ BEFORE INSERT OR UPDATE ON processed_messages { await cmd.ExecuteNonQueryAsync(cancellationToken); } + // Create index on processed_messages for specified columns await using (var cmd = new NpgsqlCommand(@" - CREATE INDEX IF NOT EXISTS idx_processed_messages_multi ON processed_messages ( - message_id, - time_sent, - receiving_endpoint_name, - critical_time, - processing_time, - delivery_time, - conversation_id, + CREATE INDEX IF NOT EXISTS idx_processed_messages_receiving_endpoint_name ON processed_messages ( + receiving_endpoint_name + );", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + await using (var cmd = new NpgsqlCommand(@" + CREATE INDEX IF NOT EXISTS idx_processed_messages_is_system_message ON processed_messages ( is_system_message );", connection)) { await cmd.ExecuteNonQueryAsync(cancellationToken); } + await using (var cmd = new NpgsqlCommand(@" + CREATE INDEX IF NOT EXISTS idx_processed_messages_by_time_sent ON processed_messages ( + time_sent + );", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + await using (var cmd = new NpgsqlCommand(@" + CREATE INDEX IF NOT EXISTS idx_processed_messages_by_critical_time ON processed_messages ( + critical_time + );", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + await using (var cmd = new NpgsqlCommand(@" + CREATE INDEX IF NOT EXISTS idx_processed_messages_by_processing_time ON processed_messages ( + processing_time + );", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + await using (var cmd = new NpgsqlCommand(@" + CREATE INDEX IF NOT EXISTS idx_processed_messages_by_delivery_time ON processed_messages ( + delivery_time + );", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + await using (var cmd = new NpgsqlCommand(@" + CREATE INDEX IF NOT EXISTS idx_processed_messages_by_message_id ON processed_messages ( + message_id + );", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + await using (var cmd = new NpgsqlCommand(@" + CREATE INDEX IF NOT EXISTS idx_processed_messages_by_conversation_id ON processed_messages ( + conversation_id + );", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + await using (var cmd = new NpgsqlCommand(@" + CREATE INDEX IF NOT EXISTS idx_processed_messages_by_created_at ON processed_messages ( + created_at + );", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + await using (var cmd = new NpgsqlCommand(@" + CREATE INDEX IF NOT EXISTS idx_processed_messages_by_query ON processed_messages ( + query + );", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + // Create saga_snapshots table await using (var cmd = new NpgsqlCommand(@" CREATE TABLE IF NOT EXISTS saga_snapshots ( From db13583cc26912561dd81ffc750acdd3394a3a54 Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 29 Aug 2025 16:00:02 +1000 Subject: [PATCH 22/41] Adding a retention cleanup background service --- .../DatabaseConfiguration.cs | 3 +- .../PostgreSQLPersistence.cs | 1 + .../RetentionCleanupService.cs | 75 +++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/DatabaseConfiguration.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/DatabaseConfiguration.cs index 41ef7619d2..bfecfe09c5 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/DatabaseConfiguration.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/DatabaseConfiguration.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; using System; + class DatabaseConfiguration( string databaseName, string adminDatabaseName, @@ -12,9 +13,7 @@ class DatabaseConfiguration( public string Name { get; } = databaseName; public string AdminDatabaseName { get; } = adminDatabaseName; public int ExpirationProcessTimerInSeconds { get; } = expirationProcessTimerInSeconds; - public TimeSpan AuditRetentionPeriod { get; } = auditRetentionPeriod; - public int MaxBodySizeToStore { get; } = maxBodySizeToStore; public string ConnectionString { get; } = connectionString; } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs index b7694ab441..cd3ddf28f7 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs @@ -23,5 +23,6 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddHostedService(); } } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs new file mode 100644 index 0000000000..5b8b5c805f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs @@ -0,0 +1,75 @@ +namespace ServiceControl.Audit.Persistence.PostgreSQL; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Npgsql; + +class RetentionCleanupService( + ILogger logger, + DatabaseConfiguration config, + PostgreSQLConnectionFactory connectionFactory) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + logger.LogInformation($"{nameof(RetentionCleanupService)} started."); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + await Task.Delay(TimeSpan.FromSeconds(config.ExpirationProcessTimerInSeconds), stoppingToken); + + await CleanupOldMessagesAsync(stoppingToken); + } + catch (OperationCanceledException) + { + // Expected when shutting down + break; + } + catch (Exception ex) + { + logger.LogError(ex, "Error during cleanup task."); + } + } + + logger.LogInformation($"{nameof(RetentionCleanupService)} stopped."); + } + + async Task CleanupOldMessagesAsync(CancellationToken cancellationToken) + { + await using var conn = await connectionFactory.OpenConnection(cancellationToken); + + var cutoffDate = DateTime.UtcNow - config.AuditRetentionPeriod; + var totalDeleted = 0; + + while (!cancellationToken.IsCancellationRequested) + { + // Delete in batches + var sql = @" + DELETE FROM processed_messages + WHERE created_at < @cutoff + LIMIT 1000;"; + + await using var cmd = new NpgsqlCommand(sql, conn); + cmd.Parameters.AddWithValue("cutoff", cutoffDate); + + var rows = await cmd.ExecuteNonQueryAsync(cancellationToken); + totalDeleted += rows; + + if (rows < 1000) + { + break; // no more rows to delete in this run + } + + await Task.Delay(TimeSpan.FromSeconds(20), cancellationToken); + } + + if (totalDeleted > 0) + { + logger.LogInformation("Deleted {Count} old messages older than {Cutoff}", totalDeleted, cutoffDate); + } + } +} From 7db54726d9e525875834a50eef2092b7054c952e Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 29 Aug 2025 16:05:29 +1000 Subject: [PATCH 23/41] Enforce body storage max size --- .../UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs | 6 ++++-- .../PostgreSQLAuditIngestionUnitOfWorkFactory.cs | 11 ++--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index 9e45db6b7b..16f89a2f00 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -14,11 +14,13 @@ class PostgreSQLAuditIngestionUnitOfWork : IAuditIngestionUnitOfWork { readonly NpgsqlBatch batch; readonly NpgsqlConnection connection; + readonly DatabaseConfiguration databaseConfiguration; - public PostgreSQLAuditIngestionUnitOfWork(NpgsqlConnection connection) + public PostgreSQLAuditIngestionUnitOfWork(NpgsqlConnection connection, DatabaseConfiguration databaseConfiguration) { batch = new NpgsqlBatch(connection); this.connection = connection; + this.databaseConfiguration = databaseConfiguration; } public async ValueTask DisposeAsync() @@ -47,7 +49,7 @@ INSERT INTO processed_messages ( );"; processedMessage.MessageMetadata["ContentLength"] = body.Length; - if (!body.IsEmpty) + if (!body.IsEmpty && body.Length <= databaseConfiguration.MaxBodySizeToStore) { cmd.Parameters.AddWithValue("body", body); } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs index e5d805cebb..0585cd98ac 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWorkFactory.cs @@ -5,19 +5,12 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; using ServiceControl.Audit.Persistence.UnitOfWork; using ServiceControl.Audit.Persistence.PostgreSQL; -class PostgreSQLAuditIngestionUnitOfWorkFactory : IAuditIngestionUnitOfWorkFactory +class PostgreSQLAuditIngestionUnitOfWorkFactory(PostgreSQLConnectionFactory connectionFactory, DatabaseConfiguration databaseConfiguration) : IAuditIngestionUnitOfWorkFactory { - readonly PostgreSQLConnectionFactory connectionFactory; - - public PostgreSQLAuditIngestionUnitOfWorkFactory(PostgreSQLConnectionFactory connectionFactory) - { - this.connectionFactory = connectionFactory; - } - public async ValueTask StartNew(int batchSize, CancellationToken cancellationToken) { var connection = await connectionFactory.OpenConnection(cancellationToken); - return new PostgreSQLAuditIngestionUnitOfWork(connection); + return new PostgreSQLAuditIngestionUnitOfWork(connection, databaseConfiguration); } public bool CanIngestMore() => true; From ecf20cc9ffb4fcacd022dc81760f7ba4069585a6 Mon Sep 17 00:00:00 2001 From: John Simons Date: Mon, 1 Sep 2025 16:46:01 +1000 Subject: [PATCH 24/41] Implemented saga logic --- .../PostgreSQLAuditDataStore.cs | 34 ++++++++++++++++++- .../PostgreSQLPersistenceInstaller.cs | 30 +++++++++------- .../PostgreSQLAuditIngestionUnitOfWork.cs | 34 ++++++++++--------- 3 files changed, 68 insertions(+), 30 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs index c8e2ae2047..3a471481c7 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs @@ -163,7 +163,39 @@ public Task>> QueryMessagesByReceivingEndpointAn return ExecuteMessagesQuery(builder, cancellationToken); } - public Task> QuerySagaHistoryById(Guid input, CancellationToken cancellationToken) => throw new NotImplementedException(); + public async Task> QuerySagaHistoryById(Guid input, CancellationToken cancellationToken) + { + await using var conn = await connectionFactory.OpenConnection(cancellationToken); + await using var cmd = new NpgsqlCommand(@" + SELECT + id, + saga_id, + saga_type, + changes + FROM saga_snapshots + WHERE saga_id = @saga_id + LIMIT 1", conn); + + cmd.Parameters.AddWithValue("saga_id", input); + + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + + if (await reader.ReadAsync(cancellationToken)) + { + var changes = GetValue>(reader, "changes") ?? []; + var sagaHistory = new SagaHistory + { + Id = GetValue(reader, "id"), + SagaId = GetValue(reader, "saga_id"), + SagaType = GetValue(reader, "saga_type"), + Changes = changes + }; + + return new QueryResult(sagaHistory, new QueryStatsInfo(string.Empty, changes.Count)); + } + + return QueryResult.Empty(); + } async Task>> ExecuteMessagesQuery( PostgresqlMessagesQueryBuilder builder, diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs index e92f6e8df5..3107333d72 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs @@ -130,17 +130,19 @@ CREATE INDEX IF NOT EXISTS idx_processed_messages_by_message_id ON processed_mes await cmd.ExecuteNonQueryAsync(cancellationToken); } + await using (var cmd = new NpgsqlCommand(@" - CREATE INDEX IF NOT EXISTS idx_processed_messages_by_conversation_id ON processed_messages ( - conversation_id + CREATE INDEX IF NOT EXISTS idx_processed_messages_by_created_at ON processed_messages ( + created_at );", connection)) { await cmd.ExecuteNonQueryAsync(cancellationToken); } await using (var cmd = new NpgsqlCommand(@" - CREATE INDEX IF NOT EXISTS idx_processed_messages_by_created_at ON processed_messages ( - created_at + CREATE INDEX IF NOT EXISTS idx_processed_messages_by_conversation ON processed_messages ( + conversation_id, + time_sent );", connection)) { await cmd.ExecuteNonQueryAsync(cancellationToken); @@ -157,17 +159,19 @@ CREATE INDEX IF NOT EXISTS idx_processed_messages_by_query ON processed_messages // Create saga_snapshots table await using (var cmd = new NpgsqlCommand(@" CREATE TABLE IF NOT EXISTS saga_snapshots ( - id TEXT PRIMARY KEY, + id UUID PRIMARY KEY, saga_id UUID, saga_type TEXT, - start_time TIMESTAMPTZ, - finish_time TIMESTAMPTZ, - status TEXT, - state_after_change TEXT, - initiating_message JSONB, - outgoing_messages JSONB, - endpoint TEXT, - processed_at TIMESTAMPTZ + changes JSONB + );", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + // Create index on saga_snapshots for faster saga_id lookups + await using (var cmd = new NpgsqlCommand(@" + CREATE INDEX IF NOT EXISTS idx_saga_snapshots_saga_id ON saga_snapshots ( + saga_id );", connection)) { await cmd.ExecuteNonQueryAsync(cancellationToken); diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index 16f89a2f00..c53668c357 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -78,27 +78,29 @@ INSERT INTO processed_messages ( public Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken) { - // Insert SagaSnapshot into saga_snapshots table + var newChange = new + { + sagaSnapshot.StartTime, + sagaSnapshot.FinishTime, + sagaSnapshot.Status, + sagaSnapshot.StateAfterChange, + sagaSnapshot.InitiatingMessage, + sagaSnapshot.OutgoingMessages, + sagaSnapshot.Endpoint + }; + + // Insert or update saga_snapshots table - add new change to the changes array var cmd = batch.CreateBatchCommand(); cmd.CommandText = @" - INSERT INTO saga_snapshots ( - id, saga_id, saga_type, start_time, finish_time, status, state_after_change, initiating_message, outgoing_messages, endpoint, processed_at - ) VALUES ( - @id, @saga_id, @saga_type, @start_time, @finish_time, @status, @state_after_change, @initiating_message, @outgoing_messages, @endpoint, @processed_at - ) - ON CONFLICT (id) DO NOTHING;"; + INSERT INTO saga_snapshots (id, saga_id, saga_type, changes) + VALUES (@saga_id, @saga_id, @saga_type, @new_change) + ON CONFLICT (id) DO UPDATE SET + changes = COALESCE(saga_snapshots.changes, '[]'::jsonb) || @new_change::jsonb;"; - cmd.Parameters.AddWithValue("id", sagaSnapshot.Id); cmd.Parameters.AddWithValue("saga_id", sagaSnapshot.SagaId); cmd.Parameters.AddWithValue("saga_type", sagaSnapshot.SagaType); - cmd.Parameters.AddWithValue("start_time", sagaSnapshot.StartTime); - cmd.Parameters.AddWithValue("finish_time", sagaSnapshot.FinishTime); - cmd.Parameters.AddWithValue("status", sagaSnapshot.Status.ToString()); - cmd.Parameters.AddWithValue("state_after_change", sagaSnapshot.StateAfterChange); - cmd.Parameters.AddWithValue("initiating_message", NpgsqlTypes.NpgsqlDbType.Jsonb, sagaSnapshot.InitiatingMessage); - cmd.Parameters.AddWithValue("outgoing_messages", NpgsqlTypes.NpgsqlDbType.Jsonb, sagaSnapshot.OutgoingMessages); - cmd.Parameters.AddWithValue("endpoint", sagaSnapshot.Endpoint); - cmd.Parameters.AddWithValue("processed_at", sagaSnapshot.ProcessedAt); + cmd.Parameters.AddWithValue("new_change", NpgsqlTypes.NpgsqlDbType.Jsonb, new[] { newChange }); + batch.BatchCommands.Add(cmd); return Task.CompletedTask; From 315cf2a0b12f85570c1a4a482aeff4f695adeecd Mon Sep 17 00:00:00 2001 From: John Simons Date: Mon, 1 Sep 2025 16:46:29 +1000 Subject: [PATCH 25/41] Fixed known_endpoints insertions --- .../UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index c53668c357..eb8f0e8ce0 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -116,7 +116,8 @@ INSERT INTO known_endpoints ( ) VALUES ( @id, @name, @host_id, @host, @last_seen ) - ON CONFLICT (id) DO NOTHING;"; + ON CONFLICT (id) DO UPDATE SET + last_seen = GREATEST(known_endpoints.last_seen, EXCLUDED.last_seen);"; cmd.Parameters.AddWithValue("id", knownEndpoint.Id); cmd.Parameters.AddWithValue("name", knownEndpoint.Name); From d7deac3a65b9f37a07f8b22cdd1875b77de96f56 Mon Sep 17 00:00:00 2001 From: John Simons Date: Mon, 1 Sep 2025 17:34:04 +1000 Subject: [PATCH 26/41] Updated retention cleanup to include saga snapshots and known endpoints --- .../PostgreSQLPersistenceInstaller.cs | 32 ++++++++++++++++--- .../RetentionCleanupService.cs | 21 +++++++++--- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs index 3107333d72..55d6b9b9cb 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs @@ -162,7 +162,8 @@ CREATE TABLE IF NOT EXISTS saga_snapshots ( id UUID PRIMARY KEY, saga_id UUID, saga_type TEXT, - changes JSONB + changes JSONB, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() );", connection)) { await cmd.ExecuteNonQueryAsync(cancellationToken); @@ -184,17 +185,38 @@ CREATE TABLE IF NOT EXISTS known_endpoints ( name TEXT, host_id UUID, host TEXT, - last_seen TIMESTAMPTZ + last_seen TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() );", connection)) { await cmd.ExecuteNonQueryAsync(cancellationToken); } + + // Create trigger to auto-update updated_at for saga_snapshots + await using (var cmd = new NpgsqlCommand(@" + CREATE OR REPLACE FUNCTION update_updated_at_column() RETURNS trigger AS $$ + BEGIN + NEW.updated_at = now(); + RETURN NEW; + END + $$ LANGUAGE plpgsql; + + DROP TRIGGER IF EXISTS saga_snapshots_updated_at_trigger ON saga_snapshots; + CREATE TRIGGER saga_snapshots_updated_at_trigger + BEFORE UPDATE ON saga_snapshots + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + + DROP TRIGGER IF EXISTS known_endpoints_updated_at_trigger ON known_endpoints; + CREATE TRIGGER known_endpoints_updated_at_trigger + BEFORE UPDATE ON known_endpoints + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } } public Task StopAsync(CancellationToken cancellationToken) { return Task.CompletedTask; } -} - - +} \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs index 5b8b5c805f..4ac55fb2f0 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs @@ -43,14 +43,27 @@ async Task CleanupOldMessagesAsync(CancellationToken cancellationToken) await using var conn = await connectionFactory.OpenConnection(cancellationToken); var cutoffDate = DateTime.UtcNow - config.AuditRetentionPeriod; + + // Cleanup processed messages + await CleanupTable("processed_messages", "created_at", cutoffDate, conn, cancellationToken); + + // Cleanup saga snapshots + await CleanupTable("saga_snapshots", "updated_at", cutoffDate, conn, cancellationToken); + + // Cleanup known endpoints + await CleanupTable("known_endpoints", "updated_at", cutoffDate, conn, cancellationToken); + } + + async Task CleanupTable(string tableName, string dateColumn, DateTime cutoffDate, NpgsqlConnection conn, CancellationToken cancellationToken) + { var totalDeleted = 0; while (!cancellationToken.IsCancellationRequested) { // Delete in batches - var sql = @" - DELETE FROM processed_messages - WHERE created_at < @cutoff + var sql = $@" + DELETE FROM {tableName} + WHERE {dateColumn} < @cutoff LIMIT 1000;"; await using var cmd = new NpgsqlCommand(sql, conn); @@ -69,7 +82,7 @@ WHERE created_at < @cutoff if (totalDeleted > 0) { - logger.LogInformation("Deleted {Count} old messages older than {Cutoff}", totalDeleted, cutoffDate); + logger.LogInformation("Deleted {Count} old records from {Table} older than {Cutoff}", totalDeleted, tableName, cutoffDate); } } } From acd267fbf4ae4148fd884f539e9277ec731f0e0e Mon Sep 17 00:00:00 2001 From: John Simons Date: Mon, 1 Sep 2025 18:25:21 +1000 Subject: [PATCH 27/41] Remove warnings --- .../PostgreSQLConnectionFactory.cs | 3 +- .../PostgreSQLAuditIngestionUnitOfWork.cs | 30 +++++++++++++------ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs index 00df7cd323..3b254897b1 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLConnectionFactory.cs @@ -3,14 +3,13 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; using Npgsql; using System.Threading.Tasks; using System.Threading; -using Microsoft.Extensions.Logging; class PostgreSQLConnectionFactory { readonly NpgsqlDataSource dataSource; readonly NpgsqlDataSource dataSourceAdmin; - public PostgreSQLConnectionFactory(DatabaseConfiguration databaseConfiguration, ILoggerFactory loggerFactory) + public PostgreSQLConnectionFactory(DatabaseConfiguration databaseConfiguration) { var dataSourceBuilder = new NpgsqlDataSourceBuilder(databaseConfiguration.ConnectionString) { diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index eb8f0e8ce0..1cd3f664ad 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -33,7 +33,19 @@ public async ValueTask DisposeAsync() public Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body, CancellationToken cancellationToken) { - T GetMetadata(string key) => processedMessage.MessageMetadata.TryGetValue(key, out var value) ? (T)value ?? default : default; + T? GetMetadata(string key) + { + if (processedMessage.MessageMetadata.TryGetValue(key, out var value)) + { + return (T?)value; + } + else + { + return default; + } + } + + object GetMetadataOrDbNull(string key) => GetMetadata(key) ?? (object)DBNull.Value; // Insert ProcessedMessage into processed_messages table var cmd = batch.CreateBatchCommand(); @@ -61,15 +73,15 @@ INSERT INTO processed_messages ( cmd.Parameters.AddWithValue("message_metadata", NpgsqlTypes.NpgsqlDbType.Jsonb, processedMessage.MessageMetadata); cmd.Parameters.AddWithValue("headers", NpgsqlTypes.NpgsqlDbType.Jsonb, processedMessage.Headers); cmd.Parameters.AddWithValue("processed_at", processedMessage.ProcessedAt); - cmd.Parameters.AddWithValue("message_id", GetMetadata("MessageId")); - cmd.Parameters.AddWithValue("message_type", GetMetadata("MessageType")); - cmd.Parameters.AddWithValue("is_system_message", GetMetadata("IsSystemMessage")); - cmd.Parameters.AddWithValue("time_sent", GetMetadata("TimeSent")); + cmd.Parameters.AddWithValue("message_id", GetMetadataOrDbNull("MessageId")); + cmd.Parameters.AddWithValue("message_type", GetMetadataOrDbNull("MessageType")); + cmd.Parameters.AddWithValue("is_system_message", GetMetadataOrDbNull("IsSystemMessage")); + cmd.Parameters.AddWithValue("time_sent", GetMetadataOrDbNull("TimeSent")); cmd.Parameters.AddWithValue("receiving_endpoint_name", GetMetadata("ReceivingEndpoint")?.Name ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("critical_time", GetMetadata("CriticalTime")); - cmd.Parameters.AddWithValue("processing_time", GetMetadata("ProcessingTime")); - cmd.Parameters.AddWithValue("delivery_time", GetMetadata("DeliveryTime")); - cmd.Parameters.AddWithValue("conversation_id", GetMetadata("ConversationId")); + cmd.Parameters.AddWithValue("critical_time", GetMetadataOrDbNull("CriticalTime")); + cmd.Parameters.AddWithValue("processing_time", GetMetadataOrDbNull("ProcessingTime")); + cmd.Parameters.AddWithValue("delivery_time", GetMetadataOrDbNull("DeliveryTime")); + cmd.Parameters.AddWithValue("conversation_id", GetMetadataOrDbNull("ConversationId")); cmd.Parameters.AddWithValue("status", (int)(GetMetadata("IsRetried") ? MessageStatus.ResolvedSuccessfully : MessageStatus.Successful)); batch.BatchCommands.Add(cmd); From f71d35429251d6fe3f4754912ae6d020413309b8 Mon Sep 17 00:00:00 2001 From: John Simons Date: Mon, 1 Sep 2025 18:41:25 +1000 Subject: [PATCH 28/41] Added missing index for audit counts --- .../PostgreSQLPersistenceInstaller.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs index 55d6b9b9cb..0828127291 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs @@ -156,6 +156,14 @@ CREATE INDEX IF NOT EXISTS idx_processed_messages_by_query ON processed_messages await cmd.ExecuteNonQueryAsync(cancellationToken); } + await using (var cmd = new NpgsqlCommand(@" + CREATE INDEX IF NOT EXISTS idx_processed_messages_audit_counts ON processed_messages ( + receiving_endpoint_name, processed_at + );", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + // Create saga_snapshots table await using (var cmd = new NpgsqlCommand(@" CREATE TABLE IF NOT EXISTS saga_snapshots ( From 69769a03be0da35c814066feb1cf50abf7e8e9e3 Mon Sep 17 00:00:00 2001 From: John Simons Date: Mon, 1 Sep 2025 18:49:08 +1000 Subject: [PATCH 29/41] Fix approval file --- ...tenceManifestTests.ApproveAuditInstanceManifests.approved.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ServiceControlInstaller.Engine.UnitTests/ApprovalFiles/PersistenceManifestTests.ApproveAuditInstanceManifests.approved.txt b/src/ServiceControlInstaller.Engine.UnitTests/ApprovalFiles/PersistenceManifestTests.ApproveAuditInstanceManifests.approved.txt index 59a50c03c0..3aa68f701c 100644 --- a/src/ServiceControlInstaller.Engine.UnitTests/ApprovalFiles/PersistenceManifestTests.ApproveAuditInstanceManifests.approved.txt +++ b/src/ServiceControlInstaller.Engine.UnitTests/ApprovalFiles/PersistenceManifestTests.ApproveAuditInstanceManifests.approved.txt @@ -1,4 +1,5 @@ [ + "PostgreSQL: PostgreSQL", "RavenDB: RavenDB", "RavenDB35: RavenDB 3.5 (Legacy)" ] \ No newline at end of file From d18baeefb2ec5cc09c6c7c49e6fbb5d5ccf3a24d Mon Sep 17 00:00:00 2001 From: John Simons Date: Tue, 2 Sep 2025 09:30:36 +1000 Subject: [PATCH 30/41] Fix casing name --- .../PostgreSQLAuditDataStore.cs | 12 ++++----- ...r.cs => PostgreSQLMessagesQueryBuilder.cs} | 25 ++++++++----------- .../AuditDeploymentPackageTests.cs | 1 + 3 files changed, 18 insertions(+), 20 deletions(-) rename src/ServiceControl.Audit.Persistence.PostgreSQL/{PostgresqlMessagesQueryBuilder.cs => PostgreSQLMessagesQueryBuilder.cs} (86%) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs index 3a471481c7..4d9ce3074a 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLAuditDataStore.cs @@ -45,7 +45,7 @@ public async Task GetMessageBody(string messageId, Cancellation public Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken) { - var builder = new PostgresqlMessagesQueryBuilder() + var builder = new PostgreSQLMessagesQueryBuilder() .WithSystemMessages(includeSystemMessages) .WithTimeSentRange(timeSentRange) .WithSorting(sortInfo) @@ -124,7 +124,7 @@ public async Task>> QueryKnownEndpoints(Ca public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) { - var builder = new PostgresqlMessagesQueryBuilder() + var builder = new PostgreSQLMessagesQueryBuilder() .WithSearch(searchParam) .WithTimeSentRange(timeSentRange) .WithSorting(sortInfo) @@ -134,7 +134,7 @@ public Task>> QueryMessages(string searchParam, public Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) { - var builder = new PostgresqlMessagesQueryBuilder() + var builder = new PostgreSQLMessagesQueryBuilder() .WithConversationId(conversationId) .WithSorting(sortInfo) .WithPaging(pagingInfo); @@ -143,7 +143,7 @@ public Task>> QueryMessagesByConversationId(stri public Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) { - var builder = new PostgresqlMessagesQueryBuilder() + var builder = new PostgreSQLMessagesQueryBuilder() .WithSystemMessages(includeSystemMessages) .WithEndpointName(endpointName) .WithTimeSentRange(timeSentRange) @@ -154,7 +154,7 @@ public Task>> QueryMessagesByReceivingEndpoint(b public Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) { - var builder = new PostgresqlMessagesQueryBuilder() + var builder = new PostgreSQLMessagesQueryBuilder() .WithSearch(keyword) .WithEndpointName(endpoint) .WithTimeSentRange(timeSentRange) @@ -198,7 +198,7 @@ FROM saga_snapshots } async Task>> ExecuteMessagesQuery( - PostgresqlMessagesQueryBuilder builder, + PostgreSQLMessagesQueryBuilder builder, CancellationToken cancellationToken) { await using var conn = await connectionFactory.OpenConnection(cancellationToken); diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgresqlMessagesQueryBuilder.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLMessagesQueryBuilder.cs similarity index 86% rename from src/ServiceControl.Audit.Persistence.PostgreSQL/PostgresqlMessagesQueryBuilder.cs rename to src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLMessagesQueryBuilder.cs index 61aa98b22d..4598e5dc4f 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgresqlMessagesQueryBuilder.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLMessagesQueryBuilder.cs @@ -1,19 +1,16 @@ - - - namespace ServiceControl.Audit.Persistence.PostgreSQL; -using System; using System.Collections.Generic; using System.Text; using Npgsql; using ServiceControl.Audit.Infrastructure; -public class PostgresqlMessagesQueryBuilder + +public class PostgreSQLMessagesQueryBuilder { readonly StringBuilder sql = new(); readonly List parameters = []; - public PostgresqlMessagesQueryBuilder() + public PostgreSQLMessagesQueryBuilder() { sql.Append(@"select unique_message_id, message_metadata, @@ -32,7 +29,7 @@ conversation_id from processed_messages where 1 = 1"); } - public PostgresqlMessagesQueryBuilder WithSystemMessages(bool? includeSystemMessages) + public PostgreSQLMessagesQueryBuilder WithSystemMessages(bool? includeSystemMessages) { if (includeSystemMessages.HasValue) { @@ -42,7 +39,7 @@ public PostgresqlMessagesQueryBuilder WithSystemMessages(bool? includeSystemMess return this; } - public PostgresqlMessagesQueryBuilder WithSearch(string? q) + public PostgreSQLMessagesQueryBuilder WithSearch(string? q) { if (!string.IsNullOrWhiteSpace(q)) { @@ -52,7 +49,7 @@ public PostgresqlMessagesQueryBuilder WithSearch(string? q) return this; } - public PostgresqlMessagesQueryBuilder WithConversationId(string? conversationId) + public PostgreSQLMessagesQueryBuilder WithConversationId(string? conversationId) { if (!string.IsNullOrWhiteSpace(conversationId)) { @@ -62,7 +59,7 @@ public PostgresqlMessagesQueryBuilder WithConversationId(string? conversationId) return this; } - public PostgresqlMessagesQueryBuilder WithMessageId(string? messageId) + public PostgreSQLMessagesQueryBuilder WithMessageId(string? messageId) { if (!string.IsNullOrWhiteSpace(messageId)) { @@ -72,7 +69,7 @@ public PostgresqlMessagesQueryBuilder WithMessageId(string? messageId) return this; } - public PostgresqlMessagesQueryBuilder WithEndpointName(string? endpointName) + public PostgreSQLMessagesQueryBuilder WithEndpointName(string? endpointName) { if (!string.IsNullOrWhiteSpace(endpointName)) { @@ -82,7 +79,7 @@ public PostgresqlMessagesQueryBuilder WithEndpointName(string? endpointName) return this; } - public PostgresqlMessagesQueryBuilder WithTimeSentRange(DateTimeRange? timeSentRange) + public PostgreSQLMessagesQueryBuilder WithTimeSentRange(DateTimeRange? timeSentRange) { if (timeSentRange?.From != null) { @@ -97,7 +94,7 @@ public PostgresqlMessagesQueryBuilder WithTimeSentRange(DateTimeRange? timeSentR return this; } - public PostgresqlMessagesQueryBuilder WithSorting(SortInfo sortInfo) + public PostgreSQLMessagesQueryBuilder WithSorting(SortInfo sortInfo) { sql.Append(" ORDER BY"); switch (sortInfo.Sort) @@ -132,7 +129,7 @@ public PostgresqlMessagesQueryBuilder WithSorting(SortInfo sortInfo) return this; } - public PostgresqlMessagesQueryBuilder WithPaging(PagingInfo pagingInfo) + public PostgreSQLMessagesQueryBuilder WithPaging(PagingInfo pagingInfo) { sql.Append($" LIMIT {pagingInfo.PageSize} OFFSET {pagingInfo.Offset};"); return this; diff --git a/src/ServiceControlInstaller.Packaging.UnitTests/AuditDeploymentPackageTests.cs b/src/ServiceControlInstaller.Packaging.UnitTests/AuditDeploymentPackageTests.cs index d6f319cacc..4d6ba4a1b5 100644 --- a/src/ServiceControlInstaller.Packaging.UnitTests/AuditDeploymentPackageTests.cs +++ b/src/ServiceControlInstaller.Packaging.UnitTests/AuditDeploymentPackageTests.cs @@ -16,6 +16,7 @@ public AuditDeploymentPackageTests() public void Should_package_storages_individually() { var expectedPersisters = new[] { + "PostgreSQL", "RavenDB35", // Still must exist, as Raven35 persistence.manifest file must be available for SCMU to understand old versions "RavenDB" }; From 9d8ee867b79338b59036e0149166df2b736974bb Mon Sep 17 00:00:00 2001 From: John Simons Date: Tue, 2 Sep 2025 09:42:08 +1000 Subject: [PATCH 31/41] Disable persistence for the installer --- .../persistence.manifest | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/persistence.manifest b/src/ServiceControl.Audit.Persistence.PostgreSQL/persistence.manifest index 8c29a40f00..bfb6a91db5 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/persistence.manifest +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/persistence.manifest @@ -1,4 +1,5 @@ { + "IsSupported": false, "Name": "PostgreSQL", "DisplayName": "PostgreSQL", "Description": "PostgreSQL ServiceControl Audit persister", From 75f593c53de4616c63c01bbb96fe330d76b8a666 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 4 Sep 2025 07:05:43 +1000 Subject: [PATCH 32/41] Using websearch_to_tsquery instead --- .../PostgreSQLMessagesQueryBuilder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLMessagesQueryBuilder.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLMessagesQueryBuilder.cs index 4598e5dc4f..4a7106d0d0 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLMessagesQueryBuilder.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLMessagesQueryBuilder.cs @@ -43,7 +43,7 @@ public PostgreSQLMessagesQueryBuilder WithSearch(string? q) { if (!string.IsNullOrWhiteSpace(q)) { - sql.Append(" and query @@ plainto_tsquery('english', @search)"); + sql.Append(" and query @@ websearch_to_tsquery('english', @search)"); parameters.Add(new NpgsqlParameter("search", q)); } return this; From c297f3ceac131a2dec7d5aa83b90d95fdbba69d5 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 4 Sep 2025 17:28:42 +1000 Subject: [PATCH 33/41] Fixed but with cleanup retention query --- .../RetentionCleanupService.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs index 4ac55fb2f0..89f4355466 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs @@ -63,8 +63,11 @@ async Task CleanupTable(string tableName, string dateColumn, DateTime cutoffDate // Delete in batches var sql = $@" DELETE FROM {tableName} - WHERE {dateColumn} < @cutoff - LIMIT 1000;"; + WHERE ctid IN ( + SELECT ctid FROM {tableName} + WHERE {dateColumn} < @cutoff + LIMIT 1000 + );"; await using var cmd = new NpgsqlCommand(sql, conn); cmd.Parameters.AddWithValue("cutoff", cutoffDate); From 432194d433a6fb10ffa42498ebc477a0cd257b65 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 4 Sep 2025 22:47:41 +1000 Subject: [PATCH 34/41] Small refactor --- .../RetentionCleanupService.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs index 89f4355466..e778c515db 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs @@ -56,6 +56,7 @@ async Task CleanupOldMessagesAsync(CancellationToken cancellationToken) async Task CleanupTable(string tableName, string dateColumn, DateTime cutoffDate, NpgsqlConnection conn, CancellationToken cancellationToken) { + const int batchSize = 1000; var totalDeleted = 0; while (!cancellationToken.IsCancellationRequested) @@ -66,7 +67,7 @@ DELETE FROM {tableName} WHERE ctid IN ( SELECT ctid FROM {tableName} WHERE {dateColumn} < @cutoff - LIMIT 1000 + LIMIT {batchSize} );"; await using var cmd = new NpgsqlCommand(sql, conn); @@ -75,7 +76,7 @@ LIMIT 1000 var rows = await cmd.ExecuteNonQueryAsync(cancellationToken); totalDeleted += rows; - if (rows < 1000) + if (rows < batchSize) { break; // no more rows to delete in this run } From ac809906d56108ef26dd87d6fb73c694797fe6f4 Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 5 Sep 2025 08:53:39 +1000 Subject: [PATCH 35/41] Add otel metrics for Npgsql --- src/Directory.Packages.props | 1 + .../PostgreSQLPersistence.cs | 4 ++++ .../RetentionCleanupService.cs | 10 ++++------ .../ServiceControl.Audit.Persistence.PostgreSQL.csproj | 1 + 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index b5ff262baa..f2fb97ce0a 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -34,6 +34,7 @@ + diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs index cd3ddf28f7..721096e146 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs @@ -1,11 +1,14 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; using Microsoft.Extensions.DependencyInjection; +using OpenTelemetry.Metrics; using ServiceControl.Audit.Auditing.BodyStorage; using ServiceControl.Audit.Persistence; using ServiceControl.Audit.Persistence.PostgreSQL.BodyStorage; using ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; using ServiceControl.Audit.Persistence.UnitOfWork; +using Npgsql; + class PostgreSQLPersistence(DatabaseConfiguration databaseConfiguration) : IPersistence { public void AddInstaller(IServiceCollection services) @@ -24,5 +27,6 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddHostedService(); + services.ConfigureOpenTelemetryMeterProvider(b => b.AddNpgsqlInstrumentation()); } } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs index e778c515db..a4bf4a4e05 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs @@ -22,11 +22,11 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await Task.Delay(TimeSpan.FromSeconds(config.ExpirationProcessTimerInSeconds), stoppingToken); - await CleanupOldMessagesAsync(stoppingToken); + await CleanupOldMessages(stoppingToken); } - catch (OperationCanceledException) + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { - // Expected when shutting down + logger.LogInformation($"{nameof(RetentionCleanupService)} stopped."); break; } catch (Exception ex) @@ -34,11 +34,9 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) logger.LogError(ex, "Error during cleanup task."); } } - - logger.LogInformation($"{nameof(RetentionCleanupService)} stopped."); } - async Task CleanupOldMessagesAsync(CancellationToken cancellationToken) + async Task CleanupOldMessages(CancellationToken cancellationToken) { await using var conn = await connectionFactory.OpenConnection(cancellationToken); diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj b/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj index 96ad791bdf..d3c42ed306 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj @@ -25,6 +25,7 @@ + \ No newline at end of file From 11c12b8d866bd4f9f0e28e485cc436da27cc394c Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 5 Sep 2025 14:03:37 +1000 Subject: [PATCH 36/41] Rollback otel for Postgres It didn't work --- src/Directory.Packages.props | 1 - .../PostgreSQLPersistence.cs | 3 --- .../ServiceControl.Audit.Persistence.PostgreSQL.csproj | 1 - 3 files changed, 5 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index f2fb97ce0a..b5ff262baa 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -34,7 +34,6 @@ - diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs index 721096e146..6f96bbdc20 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs @@ -1,13 +1,11 @@ namespace ServiceControl.Audit.Persistence.PostgreSQL; using Microsoft.Extensions.DependencyInjection; -using OpenTelemetry.Metrics; using ServiceControl.Audit.Auditing.BodyStorage; using ServiceControl.Audit.Persistence; using ServiceControl.Audit.Persistence.PostgreSQL.BodyStorage; using ServiceControl.Audit.Persistence.PostgreSQL.UnitOfWork; using ServiceControl.Audit.Persistence.UnitOfWork; -using Npgsql; class PostgreSQLPersistence(DatabaseConfiguration databaseConfiguration) : IPersistence { @@ -27,6 +25,5 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddHostedService(); - services.ConfigureOpenTelemetryMeterProvider(b => b.AddNpgsqlInstrumentation()); } } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj b/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj index d3c42ed306..96ad791bdf 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/ServiceControl.Audit.Persistence.PostgreSQL.csproj @@ -25,7 +25,6 @@ - \ No newline at end of file From f247ce39599c57fd0a4c6167db4cc26cf06266fb Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 5 Sep 2025 15:20:50 +1000 Subject: [PATCH 37/41] Prevent deadlocks --- .../UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index 1cd3f664ad..8969e6ee6c 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -104,6 +104,7 @@ public Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken canc // Insert or update saga_snapshots table - add new change to the changes array var cmd = batch.CreateBatchCommand(); cmd.CommandText = @" + SELECT pg_advisory_xact_lock(hashtext(@saga_id)); INSERT INTO saga_snapshots (id, saga_id, saga_type, changes) VALUES (@saga_id, @saga_id, @saga_type, @new_change) ON CONFLICT (id) DO UPDATE SET @@ -123,6 +124,7 @@ public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken c // Insert KnownEndpoint into known_endpoints table var cmd = batch.CreateBatchCommand(); cmd.CommandText = @" + SELECT pg_advisory_xact_lock(hashtext(@id)); INSERT INTO known_endpoints ( id, name, host_id, host, last_seen ) VALUES ( From 249d732c7831a1ce2527d3f9a626b17e2f478527 Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 5 Sep 2025 15:44:15 +1000 Subject: [PATCH 38/41] Try this way --- .../PostgreSQLAuditIngestionUnitOfWork.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index 8969e6ee6c..07d4851501 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -102,13 +102,17 @@ public Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken canc }; // Insert or update saga_snapshots table - add new change to the changes array + var lockCmd = batch.CreateBatchCommand(); + lockCmd.CommandText = "SELECT pg_advisory_xact_lock(hashtext(@saga_id))"; + lockCmd.Parameters.AddWithValue("saga_id", sagaSnapshot.SagaId); + batch.BatchCommands.Add(lockCmd); + var cmd = batch.CreateBatchCommand(); cmd.CommandText = @" - SELECT pg_advisory_xact_lock(hashtext(@saga_id)); INSERT INTO saga_snapshots (id, saga_id, saga_type, changes) VALUES (@saga_id, @saga_id, @saga_type, @new_change) ON CONFLICT (id) DO UPDATE SET - changes = COALESCE(saga_snapshots.changes, '[]'::jsonb) || @new_change::jsonb;"; + changes = COALESCE(saga_snapshots.changes, '[]'::jsonb) || @new_change::jsonb"; cmd.Parameters.AddWithValue("saga_id", sagaSnapshot.SagaId); cmd.Parameters.AddWithValue("saga_type", sagaSnapshot.SagaType); @@ -122,16 +126,20 @@ ON CONFLICT (id) DO UPDATE SET public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken) { // Insert KnownEndpoint into known_endpoints table + var lockCmd = batch.CreateBatchCommand(); + lockCmd.CommandText = "SELECT pg_advisory_xact_lock(hashtext(@id))"; + lockCmd.Parameters.AddWithValue("id", knownEndpoint.Id); + batch.BatchCommands.Add(lockCmd); + var cmd = batch.CreateBatchCommand(); cmd.CommandText = @" - SELECT pg_advisory_xact_lock(hashtext(@id)); INSERT INTO known_endpoints ( id, name, host_id, host, last_seen ) VALUES ( @id, @name, @host_id, @host, @last_seen ) ON CONFLICT (id) DO UPDATE SET - last_seen = GREATEST(known_endpoints.last_seen, EXCLUDED.last_seen);"; + last_seen = GREATEST(known_endpoints.last_seen, EXCLUDED.last_seen)"; cmd.Parameters.AddWithValue("id", knownEndpoint.Id); cmd.Parameters.AddWithValue("name", knownEndpoint.Name); From 3334aa3cd55031c3d57b7c331b29718a7826c654 Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 5 Sep 2025 15:50:20 +1000 Subject: [PATCH 39/41] Ensure connection is disposed --- .../PostgreSQLAuditIngestionUnitOfWork.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index 07d4851501..1176d2e108 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -25,10 +25,16 @@ public PostgreSQLAuditIngestionUnitOfWork(NpgsqlConnection connection, DatabaseC public async ValueTask DisposeAsync() { - await batch.PrepareAsync(); - await batch.ExecuteNonQueryAsync(); - await batch.DisposeAsync(); - await connection.DisposeAsync(); + try + { + await batch.PrepareAsync(); + await batch.ExecuteNonQueryAsync(); + } + finally + { + await batch.DisposeAsync(); + await connection.DisposeAsync(); + } } public Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body, CancellationToken cancellationToken) From 892799ad9e0284c78e037a161f2bfbefd86f8d98 Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 5 Sep 2025 15:58:54 +1000 Subject: [PATCH 40/41] Roll this back --- .../PostgreSQLAuditIngestionUnitOfWork.cs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index 1176d2e108..eeb376973a 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -108,17 +108,12 @@ public Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken canc }; // Insert or update saga_snapshots table - add new change to the changes array - var lockCmd = batch.CreateBatchCommand(); - lockCmd.CommandText = "SELECT pg_advisory_xact_lock(hashtext(@saga_id))"; - lockCmd.Parameters.AddWithValue("saga_id", sagaSnapshot.SagaId); - batch.BatchCommands.Add(lockCmd); - var cmd = batch.CreateBatchCommand(); cmd.CommandText = @" INSERT INTO saga_snapshots (id, saga_id, saga_type, changes) VALUES (@saga_id, @saga_id, @saga_type, @new_change) ON CONFLICT (id) DO UPDATE SET - changes = COALESCE(saga_snapshots.changes, '[]'::jsonb) || @new_change::jsonb"; + changes = COALESCE(saga_snapshots.changes, '[]'::jsonb) || @new_change::jsonb;"; cmd.Parameters.AddWithValue("saga_id", sagaSnapshot.SagaId); cmd.Parameters.AddWithValue("saga_type", sagaSnapshot.SagaType); @@ -132,11 +127,6 @@ ON CONFLICT (id) DO UPDATE SET public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken) { // Insert KnownEndpoint into known_endpoints table - var lockCmd = batch.CreateBatchCommand(); - lockCmd.CommandText = "SELECT pg_advisory_xact_lock(hashtext(@id))"; - lockCmd.Parameters.AddWithValue("id", knownEndpoint.Id); - batch.BatchCommands.Add(lockCmd); - var cmd = batch.CreateBatchCommand(); cmd.CommandText = @" INSERT INTO known_endpoints ( @@ -145,7 +135,7 @@ INSERT INTO known_endpoints ( @id, @name, @host_id, @host, @last_seen ) ON CONFLICT (id) DO UPDATE SET - last_seen = GREATEST(known_endpoints.last_seen, EXCLUDED.last_seen)"; + last_seen = GREATEST(known_endpoints.last_seen, EXCLUDED.last_seen);"; cmd.Parameters.AddWithValue("id", knownEndpoint.Id); cmd.Parameters.AddWithValue("name", knownEndpoint.Name); From c502fabb30bd12966e6dfdaa01cd6d30a5adbd20 Mon Sep 17 00:00:00 2001 From: John Simons Date: Mon, 8 Sep 2025 12:41:35 +1000 Subject: [PATCH 41/41] Made known_endpoint table insert only --- .../PostgreSQLPersistence.cs | 1 + .../PostgreSQLPersistenceInstaller.cs | 22 ++++--- .../RetentionCleanupService.cs | 7 ++- .../PostgreSQLAuditIngestionUnitOfWork.cs | 13 ++-- .../UpdateKnownEndpointTable.cs | 61 +++++++++++++++++++ 5 files changed, 87 insertions(+), 17 deletions(-) create mode 100644 src/ServiceControl.Audit.Persistence.PostgreSQL/UpdateKnownEndpointTable.cs diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs index 6f96bbdc20..f9ef859af5 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistence.cs @@ -25,5 +25,6 @@ public void AddPersistence(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddHostedService(); + services.AddHostedService(); } } diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs index 0828127291..2b5bdd360b 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/PostgreSQLPersistenceInstaller.cs @@ -186,6 +186,20 @@ CREATE INDEX IF NOT EXISTS idx_saga_snapshots_saga_id ON saga_snapshots ( await cmd.ExecuteNonQueryAsync(cancellationToken); } + // Create known_endpoints table + await using (var cmd = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS known_endpoints_insert ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + endpoint_id TEXT, + name TEXT, + host_id UUID, + host TEXT, + last_seen TIMESTAMPTZ + );", connection)) + { + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + // Create known_endpoints table await using (var cmd = new NpgsqlCommand(@" CREATE TABLE IF NOT EXISTS known_endpoints ( @@ -193,8 +207,7 @@ CREATE TABLE IF NOT EXISTS known_endpoints ( name TEXT, host_id UUID, host TEXT, - last_seen TIMESTAMPTZ, - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + last_seen TIMESTAMPTZ );", connection)) { await cmd.ExecuteNonQueryAsync(cancellationToken); @@ -212,11 +225,6 @@ CREATE OR REPLACE FUNCTION update_updated_at_column() RETURNS trigger AS $$ DROP TRIGGER IF EXISTS saga_snapshots_updated_at_trigger ON saga_snapshots; CREATE TRIGGER saga_snapshots_updated_at_trigger BEFORE UPDATE ON saga_snapshots - FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); - - DROP TRIGGER IF EXISTS known_endpoints_updated_at_trigger ON known_endpoints; - CREATE TRIGGER known_endpoints_updated_at_trigger - BEFORE UPDATE ON known_endpoints FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();", connection)) { await cmd.ExecuteNonQueryAsync(cancellationToken); diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs index a4bf4a4e05..5022c8410d 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/RetentionCleanupService.cs @@ -49,7 +49,7 @@ async Task CleanupOldMessages(CancellationToken cancellationToken) await CleanupTable("saga_snapshots", "updated_at", cutoffDate, conn, cancellationToken); // Cleanup known endpoints - await CleanupTable("known_endpoints", "updated_at", cutoffDate, conn, cancellationToken); + await CleanupTable("known_endpoints", "last_seen", cutoffDate, conn, cancellationToken); } async Task CleanupTable(string tableName, string dateColumn, DateTime cutoffDate, NpgsqlConnection conn, CancellationToken cancellationToken) @@ -59,10 +59,11 @@ async Task CleanupTable(string tableName, string dateColumn, DateTime cutoffDate while (!cancellationToken.IsCancellationRequested) { - // Delete in batches + // Delete in batches - skip if another process is already cleaning this table var sql = $@" DELETE FROM {tableName} - WHERE ctid IN ( + WHERE pg_try_advisory_xact_lock(hashtext('{tableName}')) + AND ctid IN ( SELECT ctid FROM {tableName} WHERE {dateColumn} < @cutoff LIMIT {batchSize} diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs index eeb376973a..587395e1ef 100644 --- a/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UnitOfWork/PostgreSQLAuditIngestionUnitOfWork.cs @@ -129,15 +129,14 @@ public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken c // Insert KnownEndpoint into known_endpoints table var cmd = batch.CreateBatchCommand(); cmd.CommandText = @" - INSERT INTO known_endpoints ( - id, name, host_id, host, last_seen + + INSERT INTO known_endpoints_insert ( + endpoint_id, name, host_id, host, last_seen ) VALUES ( - @id, @name, @host_id, @host, @last_seen - ) - ON CONFLICT (id) DO UPDATE SET - last_seen = GREATEST(known_endpoints.last_seen, EXCLUDED.last_seen);"; + @endpoint_id, @name, @host_id, @host, @last_seen + );"; - cmd.Parameters.AddWithValue("id", knownEndpoint.Id); + cmd.Parameters.AddWithValue("endpoint_id", knownEndpoint.Id); cmd.Parameters.AddWithValue("name", knownEndpoint.Name); cmd.Parameters.AddWithValue("host_id", knownEndpoint.HostId); cmd.Parameters.AddWithValue("host", knownEndpoint.Host); diff --git a/src/ServiceControl.Audit.Persistence.PostgreSQL/UpdateKnownEndpointTable.cs b/src/ServiceControl.Audit.Persistence.PostgreSQL/UpdateKnownEndpointTable.cs new file mode 100644 index 0000000000..1945e8cff1 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.PostgreSQL/UpdateKnownEndpointTable.cs @@ -0,0 +1,61 @@ +namespace ServiceControl.Audit.Persistence.PostgreSQL; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Npgsql; + +class UpdateKnownEndpointTable( + ILogger logger, + PostgreSQLConnectionFactory connectionFactory) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + logger.LogInformation($"{nameof(UpdateKnownEndpointTable)} started."); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + await Task.Delay(TimeSpan.FromSeconds(30), stoppingToken); + + await UpdateTable(stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + logger.LogInformation($"{nameof(UpdateKnownEndpointTable)} stopped."); + break; + } + catch (Exception ex) + { + logger.LogError(ex, "Error during update known_endpoints table."); + } + } + } + + async Task UpdateTable(CancellationToken stoppingToken) + { + await using var conn = await connectionFactory.OpenConnection(stoppingToken); + + var sql = @" + DO $$ + BEGIN + IF pg_try_advisory_xact_lock(hashtext('known_endpoints_sync')) THEN + INSERT INTO known_endpoints (id, name, host_id, host, last_seen) + SELECT DISTINCT ON (endpoint_id) endpoint_id, name, host_id, host, last_seen + FROM known_endpoints_insert + ORDER BY endpoint_id, last_seen DESC + ON CONFLICT (id) DO UPDATE SET + last_seen = GREATEST(known_endpoints.last_seen, EXCLUDED.last_seen); + + DELETE FROM known_endpoints_insert; + END IF; + END $$; + "; + + await using var cmd = new NpgsqlCommand(sql, conn); + await cmd.ExecuteNonQueryAsync(stoppingToken); + } +}