Align metrics with workflow semconv: use duration{state=pending}, move trigger.type to workflow-level metrics, add workflow.count

Author: gorbach

src/OpenTelemetry.Instrumentation.Hangfire/HangfireInstrumentationOptions.cs

@@ -47,12 +47,14 @@ public class HangfireInstrumentationOptions
     public Func<BackgroundJob, bool>? Filter { get; set; }
 
     /// <summary>
-    /// Gets or sets a value indicating whether to record queue latency metrics.
+    /// Gets or sets a value indicating whether to record the pending state duration in metrics.
     /// </summary>
     /// <remarks>
-    /// When enabled, records the time jobs spend waiting in the queue before execution.
+    /// When enabled, records workflow.execution.duration with state="pending", representing
+    /// the time jobs spend waiting in the queue before execution starts.
     /// This requires an additional database call per job execution to retrieve the enqueue timestamp.
     /// Default is <see langword="false"/> to avoid performance impact in high-throughput scenarios.
+    /// When disabled, only execution duration (state="executing") is recorded.
     /// </remarks>
     public bool RecordQueueLatency { get; set; }
 }

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetrics.cs

@@ -12,18 +12,18 @@ namespace OpenTelemetry.Instrumentation.Hangfire.Implementation;
 /// </summary>
 internal static class HangfireMetrics
 {
-    private static readonly Assembly Assembly = typeof(HangfireMetrics).Assembly;
-    private static readonly AssemblyName AssemblyName = Assembly.GetName();
-    internal static readonly string MeterName = AssemblyName.Name!;
-    private static readonly string InstrumentationVersion = Assembly.GetPackageVersion();
-
     // Metric name constants
     internal const string ExecutionCountMetricName = "workflow.execution.count";
     internal const string ExecutionDurationMetricName = "workflow.execution.duration";
     internal const string ExecutionStatusMetricName = "workflow.execution.status";
     internal const string ExecutionErrorsMetricName = "workflow.execution.errors";
+    internal const string WorkflowCountMetricName = "workflow.count";
 
-    internal const string QueueLatencyMetricName = "hangfire.queue.latency";
+    internal static readonly Assembly Assembly = typeof(HangfireMetrics).Assembly;
+    internal static readonly AssemblyName AssemblyName = Assembly.GetName();
+    internal static readonly string MeterName = AssemblyName.Name!;
+
+    private static readonly string InstrumentationVersion = Assembly.GetPackageVersion();
 
     /// <summary>
     /// The meter instance for all Hangfire metrics.
@@ -35,38 +35,52 @@ internal static class HangfireMetrics
     /// Follows OpenTelemetry workflow semantic conventions.
     /// </summary>
     public static readonly Counter<long> ExecutionCount =
-        Meter.CreateCounter<long>(ExecutionCountMetricName, unit: "{executions}",
+        Meter.CreateCounter<long>(
+            ExecutionCountMetricName,
+            unit: "{executions}",
             description: "The number of task executions which have been initiated.");
 
     /// <summary>
     /// Histogram for duration of an execution grouped by task, type and result.
     /// Follows OpenTelemetry workflow semantic conventions.
+    /// Records duration for different execution phases using workflow.execution.state attribute:
+    /// - state=pending: Time spent waiting in queue before execution.
+    /// - state=executing: Time spent in actual execution.
     /// </summary>
     public static readonly Histogram<double> ExecutionDuration =
-        Meter.CreateHistogram<double>(ExecutionDurationMetricName, unit: "s",
+        Meter.CreateHistogram<double>(
+            ExecutionDurationMetricName,
+            unit: "s",
             description: "Duration of an execution grouped by task, type and result.");
 
     /// <summary>
     /// UpDownCounter for the number of actively running tasks grouped by task and state.
     /// Follows OpenTelemetry workflow semantic conventions.
     /// </summary>
     public static readonly UpDownCounter<long> ExecutionStatus =
-        Meter.CreateUpDownCounter<long>(ExecutionStatusMetricName, unit: "{executions}",
+        Meter.CreateUpDownCounter<long>(
+            ExecutionStatusMetricName,
+            unit: "{executions}",
             description: "The number of actively running tasks grouped by task, type and the current state.");
 
     /// <summary>
     /// Counter for the number of errors encountered in task runs (eg. compile, test failures).
     /// Follows OpenTelemetry workflow semantic conventions.
     /// </summary>
     public static readonly Counter<long> ExecutionErrors =
-        Meter.CreateCounter<long>(ExecutionErrorsMetricName, unit: "{error}",
+        Meter.CreateCounter<long>(
+            ExecutionErrorsMetricName,
+            unit: "{error}",
             description: "The number of errors encountered in task runs (eg. compile, test failures).");
 
     /// <summary>
-    /// Histogram for time tasks spend waiting in queue before execution.
-    /// Hangfire-specific metric (not part of standard workflow conventions).
+    /// Counter for the number of workflow instances which have been initiated.
+    /// Follows OpenTelemetry workflow semantic conventions.
+    /// In Hangfire, this tracks individual job completions. For batch workflows, this would track batch completion.
     /// </summary>
-    public static readonly Histogram<double> QueueLatency =
-        Meter.CreateHistogram<double>(QueueLatencyMetricName, unit: "s",
-            description: "Time tasks spend waiting in queue before execution starts.");
+    public static readonly Counter<long> WorkflowCount =
+        Meter.CreateCounter<long>(
+            WorkflowCountMetricName,
+            unit: "{workflows}",
+            description: "The number of workflow instances which have been initiated.");
 }

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsInstrumentation.cs

@@ -19,10 +19,10 @@ public HangfireMetricsInstrumentation(HangfireInstrumentationOptions options)
         this.AddFilter(new HangfireMetricsStateFilter());
         this.AddFilter(new HangfireMetricsErrorFilterAttribute());
 
-        // Only register queue latency filter if enabled (requires DB call per job)
+        // Only register pending duration filter if enabled (requires DB call per job)
         if (options.RecordQueueLatency)
         {
-            this.AddFilter(new HangfireQueueLatencyFilterAttribute());
+            this.AddFilter(new HangfirePendingDurationFilterAttribute());
         }
     }
 

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsJobFilterAttribute.cs

@@ -21,16 +21,46 @@ public void OnPerforming(PerformingContext performingContext)
 
     public void OnPerformed(PerformedContext performedContext)
     {
-        var executionTags = HangfireTagBuilder.BuildExecutionTags(performedContext.BackgroundJob.Job, performedContext.Exception);
+        // Get recurring job ID if this job was triggered by a recurring job
+        string? recurringJobId = null;
+        try
+        {
+            recurringJobId = performedContext.Connection.GetJobParameter(
+                performedContext.BackgroundJob.Id,
+                "RecurringJobId");
+        }
+        catch
+        {
+            // If we can't get the recurring job ID, treat it as a non-recurring job
+        }
 
-        HangfireMetrics.ExecutionCount.Add(1, executionTags);
+        // Record execution count (without state attribute per semantic conventions)
+        var countTags = HangfireTagBuilder.BuildExecutionCountTags(
+            performedContext.BackgroundJob.Job,
+            performedContext.Exception);
 
+        HangfireMetrics.ExecutionCount.Add(1, countTags);
+
+        // Record execution duration (with state="executing" to differentiate from pending phase)
         if (performedContext.Items.TryGetValue(StopwatchKey, out var stopwatchObj) && stopwatchObj is Stopwatch stopwatch)
         {
             stopwatch.Stop();
             var duration = stopwatch.Elapsed.TotalSeconds;
 
-            HangfireMetrics.ExecutionDuration.Record(duration, executionTags);
+            var durationTags = HangfireTagBuilder.BuildExecutionTags(
+                performedContext.BackgroundJob.Job,
+                performedContext.Exception,
+                workflowState: HangfireTagBuilder.StateExecuting);
+
+            HangfireMetrics.ExecutionDuration.Record(duration, durationTags);
         }
+
+        // Record workflow-level metrics (includes trigger type)
+        var workflowTags = HangfireTagBuilder.BuildWorkflowTags(
+            performedContext.BackgroundJob.Job,
+            performedContext.Exception,
+            recurringJobId);
+
+        HangfireMetrics.WorkflowCount.Add(1, workflowTags);
     }
 }

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsStateFilter.cs

@@ -21,12 +21,10 @@ public void OnStateApplied(ApplyStateContext context, IWriteOnlyTransaction tran
         }
 
         var errorType = GetErrorTypeFromNewState(context.NewState);
-        var recurringJobId = GetRecurringJobId(context);
         var tags = HangfireTagBuilder.BuildStateTags(
             context.BackgroundJob.Job,
             workflowState,
-            errorType,
-            recurringJobId);
+            errorType);
 
         HangfireMetrics.ExecutionStatus.Add(1, tags);
     }
@@ -40,12 +38,10 @@ public void OnStateUnapplied(ApplyStateContext context, IWriteOnlyTransaction tr
         }
 
         var errorType = GetErrorTypeFromOldState(context);
-        var recurringJobId = GetRecurringJobId(context);
         var tags = HangfireTagBuilder.BuildStateTags(
             context.BackgroundJob.Job,
             workflowState,
-            errorType,
-            recurringJobId);
+            errorType);
 
         HangfireMetrics.ExecutionStatus.Add(-1, tags);
     }
@@ -91,17 +87,4 @@ public void OnStateUnapplied(ApplyStateContext context, IWriteOnlyTransaction tr
 
         return null;
     }
-
-    private static string? GetRecurringJobId(ApplyStateContext context)
-    {
-        try
-        {
-            return context.Connection.GetJobParameter(context.BackgroundJob.Id, "RecurringJobId");
-        }
-        catch
-        {
-            // Parameter doesn't exist or couldn't be retrieved
-            return null;
-        }
-    }
 }

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireQueueLatencyFilterAttribute.cs renamed to src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfirePendingDurationFilterAttribute.cs

@@ -9,13 +9,14 @@
 namespace OpenTelemetry.Instrumentation.Hangfire.Implementation;
 
 /// <summary>
-/// Hangfire filter that records queue latency metrics.
+/// Hangfire filter that records workflow.execution.duration metric for the pending state.
 /// </summary>
 /// <remarks>
 /// This filter captures the EnqueuedAt timestamp when a job enters the Enqueued state
-/// and calculates queue latency when the job starts executing.
+/// and records workflow.execution.duration{state="pending"} when the job starts executing.
+/// This represents the time a job spent waiting in the queue before execution started.
 /// </remarks>
-internal sealed class HangfireQueueLatencyFilterAttribute : JobFilterAttribute, IServerFilter, IElectStateFilter
+internal sealed class HangfirePendingDurationFilterAttribute : JobFilterAttribute, IServerFilter, IElectStateFilter
 {
     private const string EnqueuedAtParameter = "OpenTelemetry.EnqueuedAt";
 
@@ -52,10 +53,14 @@ public void OnPerforming(PerformingContext performingContext)
             if (!string.IsNullOrEmpty(enqueuedAtStr))
             {
                 var enqueuedAt = JobHelper.DeserializeDateTime(enqueuedAtStr);
-                var queueLatency = (DateTime.UtcNow - enqueuedAt).TotalSeconds;
+                var pendingDuration = (DateTime.UtcNow - enqueuedAt).TotalSeconds;
 
-                var tags = HangfireTagBuilder.BuildCommonTags(performingContext.BackgroundJob.Job);
-                HangfireMetrics.QueueLatency.Record(queueLatency, tags);
+                // Record workflow.execution.duration with state="pending"
+                var tags = HangfireTagBuilder.BuildExecutionTags(
+                    performingContext.BackgroundJob.Job,
+                    exception: null,
+                    workflowState: HangfireTagBuilder.StatePending);
+                HangfireMetrics.ExecutionDuration.Record(pendingDuration, tags);
             }
         }
         catch
@@ -66,6 +71,6 @@ public void OnPerforming(PerformingContext performingContext)
 
     public void OnPerformed(PerformedContext performedContext)
     {
-        // No-op: This filter only handles queue latency in OnPerforming
+        // No-op: This filter only handles pending state duration in OnPerforming
     }
 }