Skip to content

Add dashboard for visualizing log data with charts and statistics #178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 6 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/Serilog.Ui.Core/AggregateDataProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -75,5 +75,9 @@ public AggregateDataProvider(IEnumerable<IDataProvider> dataProviders)
/// <inheritdoc/>
public Task<(IEnumerable<LogModel>, int)> FetchDataAsync(FetchLogsQuery queryParams, CancellationToken cancellationToken = default)
=> SelectedDataProvider.FetchDataAsync(queryParams, cancellationToken);

/// <inheritdoc/>
public Task<DashboardModel> FetchDashboardAsync(CancellationToken cancellationToken = default)
=> SelectedDataProvider.FetchDashboardAsync(cancellationToken);
}
}
5 changes: 5 additions & 0 deletions src/Serilog.Ui.Core/IDataProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@ public interface IDataProvider
/// </summary>
Task<(IEnumerable<LogModel> results, int total)> FetchDataAsync(FetchLogsQuery queryParams, CancellationToken cancellationToken = default);

/// <summary>
/// Fetches dashboard statistics asynchronous.
/// </summary>
Task<DashboardModel> FetchDashboardAsync(CancellationToken cancellationToken = default);

/// <summary>
/// Name of the provider, used to identify this provider when using multiple.
/// </summary>
Expand Down
30 changes: 30 additions & 0 deletions src/Serilog.Ui.Core/Models/DashboardModel.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
using System.Collections.Generic;

namespace Serilog.Ui.Core.Models
{
/// <summary>
/// Represents dashboard statistics for log data visualization.
/// </summary>
public class DashboardModel
{
/// <summary>
/// Gets or sets the total count of logs.
/// </summary>
public int TotalLogs { get; set; }

/// <summary>
/// Gets or sets the count of logs by level.
/// </summary>
public Dictionary<string, int> LogsByLevel { get; set; } = new();

/// <summary>
/// Gets or sets the count of logs for today.
/// </summary>
public int TodayLogs { get; set; }

/// <summary>
/// Gets or sets the count of error logs for today.
/// </summary>
public int TodayErrorLogs { get; set; }
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -56,4 +56,54 @@ public class ElasticSearchDbDataProvider(IElasticClient client, ElasticSearchDbO

return (result?.Documents.Select((x, index) => x.ToLogModel(rowNoStart, index)).ToList() ?? [], total);
}

public async Task<DashboardModel> FetchDashboardAsync(CancellationToken cancellationToken = default)
{
var dashboard = new DashboardModel();
var today = DateTime.Today;
var tomorrow = today.AddDays(1);

// Get total logs count
var totalResponse = await _client.CountAsync<ElasticSearchDbLogModel>(c => c
.Index(options.IndexName), cancellationToken);
dashboard.TotalLogs = (int)(totalResponse?.Count ?? 0);

// Get logs count by level
var levelResponse = await _client.SearchAsync<ElasticSearchDbLogModel>(s => s
.Index(options.IndexName)
.Size(0)
.Aggregations(aggs => aggs
.Terms("levels", t => t.Field(f => f.Level))
), cancellationToken);

if (levelResponse?.Aggregations?.Terms("levels") is { } levelsAgg)
{
dashboard.LogsByLevel = levelsAgg.Buckets.ToDictionary(
bucket => bucket.Key.ToString() ?? "Unknown",
bucket => (int)bucket.DocCount);
}

// Get today's logs count
var todayResponse = await _client.CountAsync<ElasticSearchDbLogModel>(c => c
.Index(options.IndexName)
.Query(q => q
.DateRange(r => r.Field(f => f.Timestamp).GreaterThanOrEquals(today).LessThan(tomorrow))
), cancellationToken);
dashboard.TodayLogs = (int)(todayResponse?.Count ?? 0);

// Get today's error logs count
var todayErrorResponse = await _client.CountAsync<ElasticSearchDbLogModel>(c => c
.Index(options.IndexName)
.Query(q => q
.Bool(b => b
.Must(
m => m.Term(t => t.Field(f => f.Level).Value("Error")),
m => m.DateRange(r => r.Field(f => f.Timestamp).GreaterThanOrEquals(today).LessThan(tomorrow))
)
)
), cancellationToken);
dashboard.TodayErrorLogs = (int)(todayErrorResponse?.Count ?? 0);

return dashboard;
}
}
33 changes: 33 additions & 0 deletions src/Serilog.Ui.MongoDbProvider/MongoDbDataProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -121,5 +121,38 @@ private static SortDefinition<MongoDbLogModel> GenerateSortClause(SortProperty s

return isDesc ? Builders<MongoDbLogModel>.Sort.Descending(sortPropertyName) : Builders<MongoDbLogModel>.Sort.Ascending(sortPropertyName);
}

public async Task<DashboardModel> FetchDashboardAsync(CancellationToken cancellationToken = default)
{
var dashboard = new DashboardModel();
var today = DateTime.Today.ToUniversalTime();
var tomorrow = today.AddDays(1);

// Get total logs count
dashboard.TotalLogs = Convert.ToInt32(await _collection.CountDocumentsAsync(Builders<MongoDbLogModel>.Filter.Empty, cancellationToken: cancellationToken));

// Get logs count by level
var levelCounts = await _collection.Aggregate()
.Group(x => x.Level, g => new { Level = g.Key, Count = g.Count() })
.ToListAsync(cancellationToken);
dashboard.LogsByLevel = levelCounts.ToDictionary(x => x.Level ?? "Unknown", x => x.Count);

// Get today's logs count
var todayFilter = Builders<MongoDbLogModel>.Filter.And(
Builders<MongoDbLogModel>.Filter.Gte(x => x.UtcTimeStamp, today),
Builders<MongoDbLogModel>.Filter.Lt(x => x.UtcTimeStamp, tomorrow)
);
dashboard.TodayLogs = Convert.ToInt32(await _collection.CountDocumentsAsync(todayFilter, cancellationToken: cancellationToken));

// Get today's error logs count
var todayErrorFilter = Builders<MongoDbLogModel>.Filter.And(
Builders<MongoDbLogModel>.Filter.Eq(x => x.Level, "Error"),
Builders<MongoDbLogModel>.Filter.Gte(x => x.UtcTimeStamp, today),
Builders<MongoDbLogModel>.Filter.Lt(x => x.UtcTimeStamp, tomorrow)
);
dashboard.TodayErrorLogs = Convert.ToInt32(await _collection.CountDocumentsAsync(todayErrorFilter, cancellationToken: cancellationToken));

return dashboard;
}
}
}
37 changes: 37 additions & 0 deletions src/Serilog.Ui.MsSqlServerProvider/SqlServerDataProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using Serilog.Ui.Core;
using Serilog.Ui.Core.Models;
using Serilog.Ui.MsSqlServerProvider.Extensions;
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
Expand Down Expand Up @@ -73,4 +74,40 @@ private async Task<int> CountLogsAsync(FetchLogsQuery queryParams)
queryParams.EndDate
});
}

public async Task<DashboardModel> FetchDashboardAsync(CancellationToken cancellationToken = default)
{
var dashboard = new DashboardModel();
var today = DateTime.Today;
var tomorrow = today.AddDays(1);

using IDbConnection connection = new SqlConnection(options.ConnectionString);

// Get total logs count
var totalQuery = $"SELECT COUNT(*) FROM [{options.Schema}].[{options.TableName}]";
dashboard.TotalLogs = await connection.QueryFirstOrDefaultAsync<int>(totalQuery);

// Get logs count by level
var levelQuery = $"SELECT [{options.ColumnNames.Level}] as Level, COUNT(*) as Count FROM [{options.Schema}].[{options.TableName}] GROUP BY [{options.ColumnNames.Level}]";
var levelCounts = await connection.QueryAsync<(string Level, int Count)>(levelQuery);
dashboard.LogsByLevel = levelCounts.ToDictionary(x => x.Level ?? "Unknown", x => x.Count);

// Get today's logs count
var todayQuery = $"SELECT COUNT(*) FROM [{options.Schema}].[{options.TableName}] WHERE [{options.ColumnNames.Timestamp}] >= @StartDate AND [{options.ColumnNames.Timestamp}] < @EndDate";
dashboard.TodayLogs = await connection.QueryFirstOrDefaultAsync<int>(todayQuery, new
{
StartDate = today,
EndDate = tomorrow
});

// Get today's error logs count
var todayErrorQuery = $"SELECT COUNT(*) FROM [{options.Schema}].[{options.TableName}] WHERE [{options.ColumnNames.Level}] = 'Error' AND [{options.ColumnNames.Timestamp}] >= @StartDate AND [{options.ColumnNames.Timestamp}] < @EndDate";
dashboard.TodayErrorLogs = await connection.QueryFirstOrDefaultAsync<int>(todayErrorQuery, new
{
StartDate = today,
EndDate = tomorrow
});

return dashboard;
}
}
36 changes: 36 additions & 0 deletions src/Serilog.Ui.MySqlProvider/Shared/DataProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -72,4 +72,40 @@ private async Task<int> CountLogsAsync(FetchLogsQuery queryParams)
queryParams.EndDate
});
}

public async Task<DashboardModel> FetchDashboardAsync(CancellationToken cancellationToken = default)
{
var dashboard = new DashboardModel();
var today = DateTime.Today;
var tomorrow = today.AddDays(1);

using MySqlConnection connection = new(options.ConnectionString);

// Get total logs count
var totalQuery = $"SELECT COUNT(*) FROM `{options.Schema}`.`{options.TableName}`";
dashboard.TotalLogs = await connection.QueryFirstOrDefaultAsync<int>(totalQuery);

// Get logs count by level
var levelQuery = $"SELECT `{options.ColumnNames.Level}` as Level, COUNT(*) as Count FROM `{options.Schema}`.`{options.TableName}` GROUP BY `{options.ColumnNames.Level}`";
var levelCounts = await connection.QueryAsync<(string Level, int Count)>(levelQuery);
dashboard.LogsByLevel = levelCounts.ToDictionary(x => x.Level ?? "Unknown", x => x.Count);

// Get today's logs count
var todayQuery = $"SELECT COUNT(*) FROM `{options.Schema}`.`{options.TableName}` WHERE `{options.ColumnNames.Timestamp}` >= @StartDate AND `{options.ColumnNames.Timestamp}` < @EndDate";
dashboard.TodayLogs = await connection.QueryFirstOrDefaultAsync<int>(todayQuery, new
{
StartDate = today,
EndDate = tomorrow
});

// Get today's error logs count
var todayErrorQuery = $"SELECT COUNT(*) FROM `{options.Schema}`.`{options.TableName}` WHERE `{options.ColumnNames.Level}` = 'Error' AND `{options.ColumnNames.Timestamp}` >= @StartDate AND `{options.ColumnNames.Timestamp}` < @EndDate";
dashboard.TodayErrorLogs = await connection.QueryFirstOrDefaultAsync<int>(todayErrorQuery, new
{
StartDate = today,
EndDate = tomorrow
});

return dashboard;
}
}
38 changes: 38 additions & 0 deletions src/Serilog.Ui.PostgreSqlProvider/PostgresDataProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,42 @@ private async Task<int> CountLogsAsync(FetchLogsQuery queryParams)
queryParams.EndDate
});
}

/// <inheritdoc/>
public async Task<DashboardModel> FetchDashboardAsync(CancellationToken cancellationToken = default)
{
var dashboard = new DashboardModel();
var today = System.DateTime.Today;
var tomorrow = today.AddDays(1);

await using NpgsqlConnection connection = new(options.ConnectionString);

// Get total logs count
var totalQuery = $"SELECT COUNT(*) FROM \"{options.Schema}\".\"{options.TableName}\"";
dashboard.TotalLogs = await connection.QueryFirstOrDefaultAsync<int>(totalQuery);

// Get logs count by level
var levelQuery = $"SELECT {options.ColumnNames.Level} as Level, COUNT(*) as Count FROM \"{options.Schema}\".\"{options.TableName}\" GROUP BY {options.ColumnNames.Level}";
var levelCounts = await connection.QueryAsync<(int Level, int Count)>(levelQuery);
dashboard.LogsByLevel = levelCounts.ToDictionary(x => LogLevelConverter.GetLevelName(x.Level.ToString()), x => x.Count);

// Get today's logs count
var todayQuery = $"SELECT COUNT(*) FROM \"{options.Schema}\".\"{options.TableName}\" WHERE \"{options.ColumnNames.Timestamp}\" >= @StartDate AND \"{options.ColumnNames.Timestamp}\" < @EndDate";
dashboard.TodayLogs = await connection.QueryFirstOrDefaultAsync<int>(todayQuery, new
{
StartDate = today,
EndDate = tomorrow
});

// Get today's error logs count (Error level = 3 in PostgreSQL)
var todayErrorQuery = $"SELECT COUNT(*) FROM \"{options.Schema}\".\"{options.TableName}\" WHERE {options.ColumnNames.Level} = @ErrorLevel AND \"{options.ColumnNames.Timestamp}\" >= @StartDate AND \"{options.ColumnNames.Timestamp}\" < @EndDate";
dashboard.TodayErrorLogs = await connection.QueryFirstOrDefaultAsync<int>(todayErrorQuery, new
{
ErrorLevel = LogLevelConverter.GetLevelValue("Error"),
StartDate = today,
EndDate = tomorrow
});

return dashboard;
}
}
32 changes: 32 additions & 0 deletions src/Serilog.Ui.RavenDbProvider/RavenDbDataProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -105,4 +105,36 @@ SortDirection sortBy
_ => query.OrderByDescending(q => q.Timestamp),
};
}

/// <inheritdoc/>
public async Task<DashboardModel> FetchDashboardAsync(CancellationToken cancellationToken = default)
{
var dashboard = new DashboardModel();
var today = DateTime.Today;
var tomorrow = today.AddDays(1);

using var session = _documentStore.OpenAsyncSession();

// Get total logs count
dashboard.TotalLogs = await session.Query<RavenDbLogModel>().CountAsync();

// Get logs count by level
var levelCounts = await session.Query<RavenDbLogModel>()
.GroupBy(x => x.Level)
.Select(g => new { Level = g.Key, Count = g.Count() })
.ToListAsync();
dashboard.LogsByLevel = levelCounts.ToDictionary(x => x.Level ?? "Unknown", x => x.Count);

// Get today's logs count
dashboard.TodayLogs = await session.Query<RavenDbLogModel>()
.Where(x => x.Timestamp >= today && x.Timestamp < tomorrow)
.CountAsync();

// Get today's error logs count
dashboard.TodayErrorLogs = await session.Query<RavenDbLogModel>()
.Where(x => x.Level == "Error" && x.Timestamp >= today && x.Timestamp < tomorrow)
.CountAsync();

return dashboard;
}
}
36 changes: 36 additions & 0 deletions src/Serilog.Ui.SqliteDataProvider/SqliteDataProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,42 @@ public class SqliteDataProvider(SqliteDbOptions options, SqliteQueryBuilder quer

public string Name => _options.GetProviderName(SqliteProviderName);

public async Task<DashboardModel> FetchDashboardAsync(CancellationToken cancellationToken = default)
{
var dashboard = new DashboardModel();
var today = DateTime.Today;
var tomorrow = today.AddDays(1);

using var connection = new SqliteConnection(_options.ConnectionString);

// Get total logs count
var totalQuery = $"SELECT COUNT(*) FROM {_options.TableName}";
dashboard.TotalLogs = await connection.QueryFirstOrDefaultAsync<int>(totalQuery);

// Get logs count by level
var levelQuery = $"SELECT {_options.ColumnNames.Level} as Level, COUNT(*) as Count FROM {_options.TableName} GROUP BY {_options.ColumnNames.Level}";
var levelCounts = await connection.QueryAsync<(string Level, int Count)>(levelQuery);
dashboard.LogsByLevel = levelCounts.ToDictionary(x => x.Level ?? "Unknown", x => x.Count);

// Get today's logs count
var todayQuery = $"SELECT COUNT(*) FROM {_options.TableName} WHERE {_options.ColumnNames.Timestamp} >= @StartDate AND {_options.ColumnNames.Timestamp} < @EndDate";
dashboard.TodayLogs = await connection.QueryFirstOrDefaultAsync<int>(todayQuery, new
{
StartDate = StringifyDate(today),
EndDate = StringifyDate(tomorrow)
});

// Get today's error logs count
var todayErrorQuery = $"SELECT COUNT(*) FROM {_options.TableName} WHERE {_options.ColumnNames.Level} = 'Error' AND {_options.ColumnNames.Timestamp} >= @StartDate AND {_options.ColumnNames.Timestamp} < @EndDate";
dashboard.TodayErrorLogs = await connection.QueryFirstOrDefaultAsync<int>(todayErrorQuery, new
{
StartDate = StringifyDate(today),
EndDate = StringifyDate(tomorrow)
});

return dashboard;
}

private async Task<IEnumerable<LogModel>> GetLogsAsync(FetchLogsQuery queryParams)
{
var query = queryBuilder.BuildFetchLogsQuery(_options.ColumnNames, _options.Schema, _options.TableName, queryParams);
Expand Down
16 changes: 11 additions & 5 deletions src/Serilog.Ui.Web/Endpoints/ISerilogUiEndpoints.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,15 @@ public interface ISerilogUiEndpoints : ISerilogUiOptionsSetter
/// <returns>A task that represents the asynchronous operation.</returns>
Task GetApiKeysAsync();

/// <summary>
/// Asynchronously retrieves logs.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
Task GetLogsAsync();
/// <summary>
/// Asynchronously retrieves logs.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
Task GetLogsAsync();

/// <summary>
/// Asynchronously retrieves dashboard statistics.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
Task GetDashboardAsync();
}
Loading