Record metrics across the board (#347)

#346: Record metrics across the board

#!components: grid-bot

~ The following record metrics now:
- execute script command and interaction
- render command and interaction
- grid bot grpc server
- admin utility
- avatar utility
- backtrace utility
- discord webhook alert manager
- script logger

~ Changed grid_ metrics to bot_
~ Fixed error with committing in client settings factory
~ Fixed web server middleware not fetching path.

Author: nikita-petko

services/grid-bot/lib/commands/Modules/Commands/ExecuteScript.cs

@@ -13,7 +13,6 @@ namespace Grid.Bot.Commands.Public;
 using System.Text.RegularExpressions;
 
 using Discord;
-using Discord.WebSocket;
 
 using Discord.Commands;
 
@@ -39,7 +38,6 @@ namespace Grid.Bot.Commands.Public;
 /// <param name="logger">The <see cref="ILogger"/>.</param>
 /// <param name="gridSettings">The <see cref="GridSettings"/>.</param>
 /// <param name="scriptsSettings">The <see cref="ScriptsSettings"/>.</param>
-/// <param name="discordClient">The <see cref="DiscordShardedClient"/>.</param>
 /// <param name="luaUtility">The <see cref="ILuaUtility"/>.</param>
 /// <param name="floodCheckerRegistry">The <see cref="IFloodCheckerRegistry"/>.</param>
 /// <param name="backtraceUtility">The <see cref="IBacktraceUtility"/>.</param>
@@ -52,7 +50,6 @@ namespace Grid.Bot.Commands.Public;
 /// - <paramref name="logger"/> cannot be null.
 /// - <paramref name="gridSettings"/> cannot be null.
 /// - <paramref name="scriptsSettings"/> cannot be null.
-/// - <paramref name="discordClient"/> cannot be null.
 /// - <paramref name="luaUtility"/> cannot be null.
 /// - <paramref name="floodCheckerRegistry"/> cannot be null.
 /// - <paramref name="backtraceUtility"/> cannot be null.
@@ -66,7 +63,6 @@ public partial class ExecuteScript(
     ILogger logger,
     GridSettings gridSettings,
     ScriptsSettings scriptsSettings,
-    DiscordShardedClient discordClient,
     ILuaUtility luaUtility,
     IFloodCheckerRegistry floodCheckerRegistry,
     IBacktraceUtility backtraceUtility,
@@ -86,7 +82,6 @@ IGridServerFileHelper gridServerFileHelper
     private readonly GridSettings _gridSettings = gridSettings ?? throw new ArgumentNullException(nameof(gridSettings));
     private readonly ScriptsSettings _scriptsSettings = scriptsSettings ?? throw new ArgumentNullException(nameof(scriptsSettings));
 
-    private readonly DiscordShardedClient _discordClient = discordClient ?? throw new ArgumentNullException(nameof(discordClient));
     private readonly ILuaUtility _luaUtility = luaUtility ?? throw new ArgumentNullException(nameof(luaUtility));
     private readonly IFloodCheckerRegistry _floodCheckerRegistry = floodCheckerRegistry ?? throw new ArgumentNullException(nameof(floodCheckerRegistry));
     private readonly IBacktraceUtility _backtraceUtility = backtraceUtility ?? throw new ArgumentNullException(nameof(backtraceUtility));
@@ -96,7 +91,6 @@ IGridServerFileHelper gridServerFileHelper
     private readonly IScriptLogger _scriptLogger = scriptLogger ?? throw new ArgumentNullException(nameof(scriptLogger));
     private readonly IGridServerFileHelper _gridServerFileHelper = gridServerFileHelper ?? throw new ArgumentNullException(nameof(gridServerFileHelper));
 
-
     [GeneratedRegex(@"```(.*?)\s(.*?)```", RegexOptions.IgnoreCase | RegexOptions.Singleline)]
     private static partial Regex CodeBlockRegex();
     [GeneratedRegex("[\"“‘”]", RegexOptions.IgnoreCase | RegexOptions.Compiled)]
@@ -112,13 +106,21 @@ protected override async Task BeforeExecuteAsync(CommandInfo command)
         if (!_adminUtility.UserIsAdmin(Context.User))
         {
             if (_floodCheckerRegistry.ScriptExecutionFloodChecker.IsFlooded())
+            {
+                ScriptExecutionPerformanceCounters.TotalScriptExecutionsBlockedByGlobalFloodChecker.Inc();
+
                 throw new ApplicationException("Too many people are using this command at once, please wait a few moments and try again.");
+            }
 
             _floodCheckerRegistry.RenderFloodChecker.UpdateCount();
 
             var perUserFloodChecker = _floodCheckerRegistry.GetPerUserScriptExecutionFloodChecker(Context.User.Id);
             if (perUserFloodChecker.IsFlooded())
+            {
+                ScriptExecutionPerformanceCounters.TotalScriptExecutionsBlockedByPerUserFloodChecker.WithLabels(Context.User.Id.ToString()).Inc();
+
                 throw new ApplicationException("You are sending execute script commands too quickly, please wait a few moments and try again.");
+            }
 
             perUserFloodChecker.UpdateCount();
         }
@@ -152,6 +154,8 @@ private static string GetCodeBlockContents(string s)
         // Check if the input matches grid syntax error
         if (GridSyntaxErrorRegex().IsMatch(input))
         {
+            ScriptExecutionPerformanceCounters.TotalScriptExecutionsWithSyntaxErrors.WithLabels("grid-server-syntax-error:metadata").Inc();
+
             var match = GridSyntaxErrorRegex().Match(input);
             var line = match.Groups[1].Value;
             var error = match.Groups[2].Value;
@@ -167,7 +171,13 @@ private static string GetCodeBlockContents(string s)
             var maxSize = _scriptsSettings.ScriptExecutionMaxFileSizeKb;
 
             if (input.Length / 1000 > maxSize)
+            {
+                ScriptExecutionPerformanceCounters.TotalScriptExecutionsWithResultsExceedingMaxSize.WithLabels(input.Length.ToString()).Inc();
+
                 return ($"The output cannot be larger than {maxSize} KiB", null);
+            }
+
+            ScriptExecutionPerformanceCounters.TotalScriptExecutionsWithResultsViaFiles.Inc();
 
             return (fileName, new MemoryStream(Encoding.UTF8.GetBytes(input)));
         }
@@ -187,7 +197,13 @@ private static string GetCodeBlockContents(string s)
             var maxSize = _scriptsSettings.ScriptExecutionMaxResultSizeKb;
 
             if (input.Length / 1000 > maxSize)
+            {
+                ScriptExecutionPerformanceCounters.TotalScriptExecutionsWithResultsExceedingMaxSize.WithLabels(input.Length.ToString()).Inc();
+
                 return ($"The result cannot be larger than {maxSize} KiB", null);
+            }
+
+            ScriptExecutionPerformanceCounters.TotalScriptExecutionsWithResultsViaFiles.Inc();
 
             return (fileName, new MemoryStream(Encoding.UTF8.GetBytes(input)));
         }
@@ -210,9 +226,17 @@ private async Task HandleResponseAsync(string result, ReturnMetadata metadata)
             .WithCurrentTimestamp();
 
         if (metadata.Success)
+        {
+            ScriptExecutionPerformanceCounters.TotalSuccessfulScriptExecutions.Inc();
+
             builder.WithColor(Color.Green);
+        }
         else
+        {
+            ScriptExecutionPerformanceCounters.TotalFailedScriptExecutionsDueToLuaError.Inc();
+
             builder.WithColor(Color.Red);
+        }
 
         var (fileNameOrOutput, outputFile) = DetermineDescription(
             metadata.Logs,
@@ -234,6 +258,8 @@ private async Task HandleResponseAsync(string result, ReturnMetadata metadata)
 
         builder.AddField("Execution Time", $"{metadata.ExecutionTime:f5}s");
 
+        ScriptExecutionPerformanceCounters.ScriptExecutionAverageExecutionTime.Observe(metadata.ExecutionTime);
+
         var attachments = new List<FileAttachment>();
         if (outputFile != null)
             attachments.Add(new(outputFile, fileNameOrOutput));
@@ -270,6 +296,8 @@ private async Task<bool> ParseLuaAsync(string input)
 
         if (errors.Any())
         {
+            ScriptExecutionPerformanceCounters.TotalScriptExecutionsWithSyntaxErrors.WithLabels("pre-parser-syntax-error").Inc();
+
             var errorString = string.Join("\n", errors.Select(err => err.ToString()));
 
             if (errorString.Length > _maxErrorLength)
@@ -303,6 +331,8 @@ private async Task<bool> ParseLuaAsync(string input)
     [Command("execute"), Summary("Execute a script via raw text."), Alias("ex", "exc", "x")]
     public async Task ExecuteScriptFromTextAsync([Remainder] string script = "")
     {
+        ScriptExecutionPerformanceCounters.TotalScriptExecutionsByUser.WithLabels(Context.User.Id.ToString()).Inc();
+
         using var _ = Context.Channel.EnterTypingState();
 
         if (string.IsNullOrWhiteSpace(script))
@@ -331,13 +361,17 @@ public async Task ExecuteScriptFromTextAsync([Remainder] string script = "")
                 return;
             }
 
+            ScriptExecutionPerformanceCounters.TotalScriptExecutionsFromFiles.WithLabels(file.Filename, file.Size.ToString()).Inc();
+
             script = await file.GetAttachmentContentsAscii();
         }
 
         script = GetCodeBlockContents(script);
 
         if (string.IsNullOrEmpty(script))
         {
+            ScriptExecutionPerformanceCounters.TotalScriptExecutionsWithNoContent.Inc();
+
             await LuaErrorAsync("There must be content within a code block!");
 
             return;
@@ -351,6 +385,8 @@ public async Task ExecuteScriptFromTextAsync([Remainder] string script = "")
 
         if (ContainsUnicode(script))
         {
+            ScriptExecutionPerformanceCounters.TotalScriptExecutionsWithUnicode.Inc();
+
             await LuaErrorAsync("Scripts can only contain ASCII characters!");
 
             return;
@@ -370,7 +406,11 @@ public async Task ExecuteScriptFromTextAsync([Remainder] string script = "")
             );
 
         if (_scriptsSettings.LuaVMEnabled) // Disable if pre-luau, or wait for the file to be updated to support pre-luau
+        {
+            ScriptExecutionPerformanceCounters.TotalScriptExecutionsUsingLuaVM.Inc();
+
             script = string.Format(_luaUtility.LuaVMTemplate, script);
+        }
 
 #if !PRE_JSON_EXECUTION
         // isAdmin allows a bypass of disabled methods and virtualized globals
@@ -430,6 +470,8 @@ public async Task ExecuteScriptFromTextAsync([Remainder] string script = "")
                 var message = ex.Message;
                 if (GridSyntaxErrorRegex().IsMatch(message))
                 {
+                    ScriptExecutionPerformanceCounters.TotalScriptExecutionsWithSyntaxErrors.WithLabels("grid-server-syntax-error:fault").Inc();
+
                     var match = GridSyntaxErrorRegex().Match(message);
                     var line = match.Groups[1].Value;
                     var error = match.Groups[2].Value;
@@ -455,6 +497,8 @@ public async Task ExecuteScriptFromTextAsync([Remainder] string script = "")
 
                 if (message.Contains(_ErrorConvertingToJson))
                 {
+                    ScriptExecutionPerformanceCounters.TotalScriptExecutionsWithNonJsonSerializableResults.Inc();
+
                     await LuaErrorAsync("The script returned a value that could not be converted to JSON.");
 
                     return;
@@ -465,18 +509,24 @@ public async Task ExecuteScriptFromTextAsync([Remainder] string script = "")
             // Catch this and alert the user (only in the case of ex is CommunicationException, ex.InnerException is InvalidOperationException and ex.InnerException.InnerException is XmlException)
             if (ex is CommunicationException && ex.InnerException is InvalidOperationException && ex.InnerException.InnerException is XmlException)
             {
+                ScriptExecutionPerformanceCounters.TotalScriptExecutionsWithNonAsciiResults.Inc();
+
                 await LuaErrorAsync("The script returned invalid ASCII characters.");
 
                 return;
             }
 
             if (ex is TimeoutException)
             {
+                ScriptExecutionPerformanceCounters.TotalScriptExecutionsThatTimedOut.Inc();
+
                 await HandleResponseAsync(null, new() { ErrorMessage = "script exceeded timeout", ExecutionTime = sw.Elapsed.TotalSeconds, Success = false });
 
                 return;
             }
 
+            ScriptExecutionPerformanceCounters.TotalScriptExecutionsWithUnexpectedExceptions.WithLabels(ex.GetType().ToString()).Inc();
+
             if (ex is not Discord.Net.HttpException)
                 await AlertForSystem(script, originalScript, scriptId, scriptName, ex);
 

services/grid-bot/lib/commands/Modules/Commands/Render.cs

@@ -48,20 +48,27 @@ IAdminUtility adminUtility
     private readonly IFloodCheckerRegistry _floodCheckerRegistry = floodCheckerRegistry ?? throw new ArgumentNullException(nameof(floodCheckerRegistry));
     private readonly IAdminUtility _adminUtility = adminUtility ?? throw new ArgumentNullException(nameof(adminUtility));
 
-
     /// <inheritdoc cref="ModuleBase{TContext}.BeforeExecuteAsync(CommandInfo)"/>
     protected override async Task BeforeExecuteAsync(CommandInfo command)
     {
         if (!_adminUtility.UserIsAdmin(Context.User))
         {
             if (_floodCheckerRegistry.RenderFloodChecker.IsFlooded())
+            {
+                RenderPerformanceCounters.TotalRendersBlockedByGlobalFloodChecker.Inc();
+
                 throw new ApplicationException("Too many people are using this command at once, please wait a few moments and try again.");
+            }
 
             _floodCheckerRegistry.RenderFloodChecker.UpdateCount();
 
             var perUserFloodChecker = _floodCheckerRegistry.GetPerUserRenderFloodChecker(Context.User.Id);
             if (perUserFloodChecker.IsFlooded())
+            {
+                RenderPerformanceCounters.TotalRendersBlockedByPerUserFloodChecker.WithLabels(Context.User.Id.ToString()).Inc();
+
                 throw new ApplicationException("You are sending render commands too quickly, please wait a few moments and try again.");
+            }
 
             perUserFloodChecker.UpdateCount();
         }
@@ -76,10 +83,14 @@ protected override async Task BeforeExecuteAsync(CommandInfo command)
     [Command("render"), Summary("Renders a Roblox character by Roblox user ID."), Alias("r")]
     public async Task DoRenderAsync(string userNameOrId)
     {
+        RenderPerformanceCounters.TotalRenders.WithLabels(userNameOrId).Inc();
+
         using var _ = Context.Channel.EnterTypingState();
 
         if (!long.TryParse(userNameOrId, out var userId))
         {
+            RenderPerformanceCounters.TotalRendersViaUsername.Inc();
+
             var id = await _rbxUsersUtility.GetUserIdByUsernameAsync(userNameOrId).ConfigureAwait(false);
             if (id == null)
             {
@@ -93,13 +104,17 @@ public async Task DoRenderAsync(string userNameOrId)
 
         if (userId < 1)
         {
+            RenderPerformanceCounters.TotalRendersWithInvalidIds.Inc();
+
             await this.ReplyWithReferenceAsync("The ID must be greater than 0.");
 
             return;
         }
 
         if (await _rbxUsersUtility.GetIsUserBannedAsync(userId).ConfigureAwait(false))
         {
+            RenderPerformanceCounters.TotalRendersAgainstBannedUsers.WithLabels(userNameOrId).Inc();
+
             _logger.Warning("The input user ID of {0} was linked to a banned user account.", userId);
             await this.ReplyWithReferenceAsync($"The user '{userNameOrId}' is banned or does not exist.");
 
@@ -127,6 +142,8 @@ public async Task DoRenderAsync(string userNameOrId)
 
             if (stream == null)
             {
+                RenderPerformanceCounters.TotalRendersWithErrors.Inc();
+
                 await this.ReplyWithReferenceAsync("An error occurred while rendering the character.");
 
                 return;
@@ -141,6 +158,8 @@ await this.ReplyWithFileAsync(
         }
         catch (ThumbnailResponseException e)
         {
+            RenderPerformanceCounters.TotalRendersWithRbxThumbnailsErrors.WithLabels(e.State.ToString()).Inc();
+
             _logger.Warning("The thumbnail service responded with the following state: {0}, message: {1}", e.State, e.Message);
 
             if (e.State == ThumbnailResponseState.InReview)
@@ -156,6 +175,8 @@ await this.ReplyWithFileAsync(
         }
         catch (Exception e)
         {
+            RenderPerformanceCounters.TotalRendersWithErrors.Inc();
+
             _logger.Error("An error occurred while rendering the character for the user '{0}': {1}", userNameOrId, e);
 
             await this.ReplyWithReferenceAsync("An error occurred while rendering the character.");