Add support for dimensions to metrics (#1447)

Author: tj-wazei

PP.md

@@ -44,14 +44,16 @@ In certain circumstances, you have the following data protection rights:
 
 The databases may store
 * `user_id` of users (the unique id of a Discord account),
+* `user_name` of users (the username of a Discord account), stored as part of metric event dimensions when tracking command usage (e.g. slash commands, tag lookups),
 * `timestamp`s of actions (for example when a command has been used),
 * `guild_id` of guilds the **bot** is member of (the unique id of a Discord guild),
 * `channel_id` of channels belonging to guilds the **bot** is member of (the unique id of a Discord channel),
 * `message_id` of messages send by users in guilds the **bot** is member of (the unique id of a Discord message),
 *  `participant_count` of no of people who participated in help thread discussions,
 *  `tags` aka categories to which these help threads belong to,
 *  `timestamp`s for both when thread was created and closed,
-*  `message_count` the no of messages that were sent in lifecycle of any help thread
+*  `message_count` the no of messages that were sent in lifecycle of any help thread,
+*  `dimensions` optional JSON metadata attached to metric events, which may include the `user_name` of the user who triggered the event and contextual details such as the command name or tag id
 
 _Note: Help threads are just threads that are created via forum channels, used for anyone to ask questions and get help
 in certain problems._

application/src/main/java/org/togetherjava/tjbot/features/analytics/EmojiTrackerListener.java

@@ -8,6 +8,8 @@
 
 import org.togetherjava.tjbot.features.MessageReceiverAdapter;
 
+import java.util.Map;
+
 /**
  * Listener that tracks custom emoji usage across all channels for analytics purposes.
  * <p>
@@ -18,6 +20,7 @@
  * custom emojis are tracked separately (e.g. {@code emoji-custom-animated-123456789}).
  */
 public final class EmojiTrackerListener extends MessageReceiverAdapter {
+    private static final String METRIC_NAME = "emoji";
     private final Metrics metrics;
 
     /**
@@ -37,7 +40,11 @@ public void onMessageReceived(MessageReceivedEvent event) {
             return;
         }
 
-        event.getMessage().getMentions().getCustomEmojis().forEach(this::trackCustomEmoji);
+        event.getMessage()
+            .getMentions()
+            .getCustomEmojis()
+            .forEach(customEmoji -> trackCustomEmoji("message", customEmoji.getIdLong(),
+                    customEmoji.isAnimated()));
     }
 
     @Override
@@ -47,11 +54,12 @@ public void onMessageReactionAdd(MessageReactionAddEvent event) {
             return;
         }
 
-        trackCustomEmoji(emoji.asCustom());
+        CustomEmoji customEmoji = emoji.asCustom();
+
+        trackCustomEmoji("reaction", customEmoji.getIdLong(), customEmoji.isAnimated());
     }
 
-    private void trackCustomEmoji(CustomEmoji emoji) {
-        String prefix = emoji.isAnimated() ? "emoji-custom-animated-" : "emoji-custom-";
-        metrics.count(prefix + emoji.getIdLong());
+    private void trackCustomEmoji(String type, long id, boolean isAnimated) {
+        metrics.count(METRIC_NAME, Map.of("type", type, "id", id, "animated", isAnimated));
     }
 }

application/src/main/java/org/togetherjava/tjbot/features/analytics/Metrics.java

@@ -1,12 +1,17 @@
 package org.togetherjava.tjbot.features.analytics;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.togetherjava.tjbot.db.Database;
 import org.togetherjava.tjbot.db.generated.tables.MetricEvents;
 
+import javax.annotation.Nullable;
+
 import java.time.Instant;
+import java.util.Map;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -15,6 +20,7 @@
  */
 public final class Metrics {
     private static final Logger logger = LoggerFactory.getLogger(Metrics.class);
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
     private final Database database;
 
@@ -35,21 +41,48 @@ public Metrics(Database database) {
      * @param event the event to save
      */
     public void count(String event) {
+        count(event, Map.of());
+    }
+
+    /**
+     * Track an event execution with additional contextual data.
+     *
+     * @param event the name of the event to record (e.g. "user_signup", "purchase")
+     * @param dimensions optional key-value pairs providing extra context about the event. These are
+     *        often referred to as "metadata" and can include things like: userId: "12345", name:
+     *        "John Smith", channel_name: "chit-chat" etc. This data helps with filtering, grouping,
+     *        and analyzing events later. Note: A value for a metric should be a Java primitive
+     *        (String, int, double, long float).
+     */
+    public void count(String event, Map<String, Object> dimensions) {
         logger.debug("Counting new record for event: {}", event);
-        Instant moment = Instant.now();
-        service.submit(() -> processEvent(event, moment));
 
+        Instant happenedAt = Instant.now();
+        String serializedDimensions = serializeDimensions(dimensions);
+
+        service.submit(() -> processEvent(event, happenedAt,
+                dimensions.isEmpty() ? null : serializedDimensions));
+    }
+
+    private static String serializeDimensions(Map<String, Object> dimensions) {
+        try {
+            return OBJECT_MAPPER.writeValueAsString(dimensions);
+        } catch (JsonProcessingException e) {
+            throw new IllegalArgumentException("Failed to serialize dimensions", e);
+        }
     }
 
     /**
      *
      * @param event the event to save
      * @param happenedAt the moment when the event is dispatched
+     * @param dimensionsJson optional JSON-serialized dimensions, or null
      */
-    private void processEvent(String event, Instant happenedAt) {
+    private void processEvent(String event, Instant happenedAt, @Nullable String dimensionsJson) {
         database.write(context -> context.newRecord(MetricEvents.METRIC_EVENTS)
             .setEvent(event)
             .setHappenedAt(happenedAt)
+            .setDimensions(dimensionsJson)
             .insert());
     }
 

application/src/main/java/org/togetherjava/tjbot/features/code/CodeMessageHandler.java

@@ -188,7 +188,7 @@ public void onButtonClick(ButtonInteractionEvent event, List<String> args) {
                 CodeFence code = extractCodeOrFallback(originalMessage.get().getContentRaw());
 
                 // Apply the selected action
-                metrics.count("code_action-" + codeAction.getLabel());
+                metrics.count("code_action", Map.of("name", codeAction.getLabel()));
                 return event.getHook()
                     .editOriginalEmbeds(codeAction.apply(code))
                     .setActionRow(createButtons(originalMessageId, codeAction));

application/src/main/java/org/togetherjava/tjbot/features/help/AutoPruneHelperRoutine.java

@@ -112,7 +112,7 @@ private void pruneRoleIfFull(List<Member> members, Role targetRole,
 
         if (isRoleFull(withRole)) {
             logger.debug("Helper role {} is full, starting to prune.", targetRole.getName());
-            metrics.count("autoprune_helper-" + targetRole.getName());
+            metrics.count("autoprune_helper", Map.of("role", targetRole.getName()));
             pruneRole(targetRole, withRole, selectRoleChannel, when);
         }
     }

application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCommand.java

@@ -162,7 +162,7 @@ private void changeCategory(SlashCommandInteractionEvent event, ThreadChannel he
         event.deferReply().queue();
         refreshCooldownFor(Subcommand.CHANGE_CATEGORY, helpThread);
 
-        metrics.count("help-category-" + category);
+        metrics.count("help-category", Map.of(CHANGE_CATEGORY_SUBCOMMAND, category));
         helper.changeChannelCategory(helpThread, category)
             .flatMap(_ -> sendCategoryChangedMessage(helpThread.getGuild(), event.getHook(),
                     helpThread, category))

application/src/main/java/org/togetherjava/tjbot/features/system/BotCore.java

@@ -385,11 +385,16 @@ public void onSlashCommandInteraction(SlashCommandInteractionEvent event) {
             SlashCommand interactor = requireUserInteractor(
                     UserInteractionType.SLASH_COMMAND.getPrefixedName(name), SlashCommand.class);
 
-            String eventName = "slash-" + name;
+            Map<String, Object> dimensions = new HashMap<>();
+            dimensions.put("name", name);
+            dimensions.put("user", event.getUser().getName());
+            dimensions.put("userId", event.getUser().getIdLong());
+
             if (event.getSubcommandName() != null) {
-                eventName += "_" + event.getSubcommandName();
+                dimensions.put("subCommandName", event.getSubcommandName());
             }
-            metrics.count(eventName);
+
+            metrics.count("slash", dimensions);
 
             interactor.onSlashCommand(event);
         });

application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java

@@ -28,6 +28,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -91,7 +92,8 @@ public void onSlashCommand(SlashCommandInteractionEvent event) {
         if (tagSystem.handleIsUnknownTag(id, event)) {
             return;
         }
-        metrics.count("tag-" + id);
+        metrics.count("tag", Map.of("id", id, "user", event.getUser().getName(), "userId",
+                event.getUser().getIdLong()));
 
         String tagContent = tagSystem.getTag(id).orElseThrow();
         MessageEmbed contentEmbed = new EmbedBuilder().setDescription(tagContent)

application/src/main/java/org/togetherjava/tjbot/features/tophelper/TopHelpersAssignmentRoutine.java

@@ -260,7 +260,7 @@ private void manageTopHelperRole(Collection<? extends Member> currentTopHelpers,
         }
 
         for (long topHelperUserId : selectedTopHelperIds) {
-            metrics.count("top_helper-" + topHelperUserId);
+            metrics.count("top_helper", Map.of("userId", topHelperUserId));
         }
         reportRoleManageSuccess(event);
     }

application/src/main/resources/db/V17__Add_Metric_Dimensions.sql

@@ -0,0 +1,41 @@
+ALTER TABLE metric_events ADD COLUMN dimensions TEXT;
+
+UPDATE metric_events
+SET event = 'code_action',
+    dimensions = json_object('name', SUBSTR(event, LENGTH('code_action-') + 1))
+WHERE event LIKE 'code_action-%';
+
+UPDATE metric_events
+SET event = 'autoprune_helper',
+    dimensions = json_object('role', SUBSTR(event, LENGTH('autoprune_helper-') + 1))
+WHERE event LIKE 'autoprune_helper-%';
+
+UPDATE metric_events
+SET event = 'help-category',
+    dimensions = json_object('category', SUBSTR(event, LENGTH('help-category-') + 1))
+WHERE event LIKE 'help-category-%';
+
+UPDATE metric_events
+SET event = 'tag',
+    dimensions = json_object('id', SUBSTR(event, LENGTH('tag-') + 1))
+WHERE event LIKE 'tag-%';
+
+UPDATE metric_events
+SET event = 'top_helper',
+    dimensions = json_object('userId', CAST(SUBSTR(event, LENGTH('top_helper-') + 1) AS INTEGER))
+WHERE event LIKE 'top_helper-%';
+
+UPDATE metric_events
+SET event = 'slash',
+    dimensions = json_object(
+        'name', SUBSTR(event, LENGTH('slash-') + 1, INSTR(SUBSTR(event, LENGTH('slash-') + 1), '_') - 1),
+        'subCommandName', SUBSTR(event, LENGTH('slash-') + 1 + INSTR(SUBSTR(event, LENGTH('slash-') + 1), '_'))
+    )
+WHERE event LIKE 'slash-%'
+  AND INSTR(SUBSTR(event, LENGTH('slash-') + 1), '_') > 0;
+
+UPDATE metric_events
+SET event = 'slash',
+    dimensions = json_object('name', SUBSTR(event, LENGTH('slash-') + 1))
+WHERE event LIKE 'slash-%'
+  AND INSTR(SUBSTR(event, LENGTH('slash-') + 1), '_') = 0;