From 2e804df251a2a6993ea09cb2801c80cff40fe154 Mon Sep 17 00:00:00 2001 From: Oleksii Gorbach Date: Mon, 20 Oct 2025 20:36:07 -0400 Subject: [PATCH 01/48] [Instrumentation.Hangfire] Add metrics support Implements OpenTelemetry workflow semantic conventions for Hangfire metrics. Fixes #2075 --- .../.publicApi/PublicAPI.Unshipped.txt | 6 + .../HangfireInstrumentationOptions.cs | 10 + .../Implementation/HangfireMetrics.cs | 72 ++++ .../HangfireMetricsErrorFilterAttribute.cs | 30 ++ .../HangfireMetricsInstrumentation.cs | 42 ++ .../HangfireMetricsJobFilterAttribute.cs | 36 ++ .../HangfireMetricsStateFilter.cs | 107 +++++ .../HangfireQueueLatencyFilterAttribute.cs | 71 ++++ .../Implementation/HangfireTagBuilder.cs | 201 +++++++++ .../MeterProviderBuilderExtensions.cs | 66 +++ ...nTelemetry.Instrumentation.Hangfire.csproj | 1 - .../README.md | 193 ++++++++- .../CustomTestException.cs | 14 + .../HangfireFixture.cs | 43 ++ ...eInstrumentationJobFilterAttributeTests.cs | 42 +- .../HangfireMetricsErrorTests.cs | 208 ++++++++++ .../HangfireMetricsTests.cs | 387 ++++++++++++++++++ .../HangfireQueueLatencyTests.cs | 225 ++++++++++ .../MetricPointExtensions.cs | 84 ++++ ...etry.Instrumentation.Hangfire.Tests.csproj | 2 - .../TestJobWithCustomException.cs | 16 + .../Utils/AssertUtils.cs | 94 +++++ .../Utils/MetricCollectionExtensions.cs | 20 + 23 files changed, 1931 insertions(+), 39 deletions(-) create mode 100644 src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetrics.cs create mode 100644 src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsErrorFilterAttribute.cs create mode 100644 src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsInstrumentation.cs create mode 100644 src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsJobFilterAttribute.cs create mode 100644 src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsStateFilter.cs create mode 100644 src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireQueueLatencyFilterAttribute.cs create mode 100644 src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireTagBuilder.cs create mode 100644 src/OpenTelemetry.Instrumentation.Hangfire/MeterProviderBuilderExtensions.cs create mode 100644 test/OpenTelemetry.Instrumentation.Hangfire.Tests/CustomTestException.cs create mode 100644 test/OpenTelemetry.Instrumentation.Hangfire.Tests/HangfireMetricsErrorTests.cs create mode 100644 test/OpenTelemetry.Instrumentation.Hangfire.Tests/HangfireMetricsTests.cs create mode 100644 test/OpenTelemetry.Instrumentation.Hangfire.Tests/HangfireQueueLatencyTests.cs create mode 100644 test/OpenTelemetry.Instrumentation.Hangfire.Tests/MetricPointExtensions.cs create mode 100644 test/OpenTelemetry.Instrumentation.Hangfire.Tests/TestJobWithCustomException.cs create mode 100644 test/OpenTelemetry.Instrumentation.Hangfire.Tests/Utils/AssertUtils.cs create mode 100644 test/OpenTelemetry.Instrumentation.Hangfire.Tests/Utils/MetricCollectionExtensions.cs diff --git a/src/OpenTelemetry.Instrumentation.Hangfire/.publicApi/PublicAPI.Unshipped.txt b/src/OpenTelemetry.Instrumentation.Hangfire/.publicApi/PublicAPI.Unshipped.txt index 7c2225b43f..e1ffb99341 100644 --- a/src/OpenTelemetry.Instrumentation.Hangfire/.publicApi/PublicAPI.Unshipped.txt +++ b/src/OpenTelemetry.Instrumentation.Hangfire/.publicApi/PublicAPI.Unshipped.txt @@ -1,3 +1,4 @@ +OpenTelemetry.Metrics.MeterProviderBuilderExtensions OpenTelemetry.Trace.HangfireInstrumentationOptions OpenTelemetry.Trace.HangfireInstrumentationOptions.DisplayNameFunc.get -> System.Func! OpenTelemetry.Trace.HangfireInstrumentationOptions.DisplayNameFunc.set -> void @@ -6,7 +7,12 @@ OpenTelemetry.Trace.HangfireInstrumentationOptions.Filter.set -> void OpenTelemetry.Trace.HangfireInstrumentationOptions.HangfireInstrumentationOptions() -> void OpenTelemetry.Trace.HangfireInstrumentationOptions.RecordException.get -> bool OpenTelemetry.Trace.HangfireInstrumentationOptions.RecordException.set -> void +OpenTelemetry.Trace.HangfireInstrumentationOptions.RecordQueueLatency.get -> bool +OpenTelemetry.Trace.HangfireInstrumentationOptions.RecordQueueLatency.set -> void OpenTelemetry.Trace.TracerProviderBuilderExtensions +static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Metrics.MeterProviderBuilder! builder) -> OpenTelemetry.Metrics.MeterProviderBuilder! +static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Metrics.MeterProviderBuilder! builder, string? name, System.Action? configure) -> OpenTelemetry.Metrics.MeterProviderBuilder! +static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Metrics.MeterProviderBuilder! builder, System.Action? configure) -> OpenTelemetry.Metrics.MeterProviderBuilder! static OpenTelemetry.Trace.TracerProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Trace.TracerProviderBuilder! builder) -> OpenTelemetry.Trace.TracerProviderBuilder! static OpenTelemetry.Trace.TracerProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Trace.TracerProviderBuilder! builder, string? name, System.Action? configure) -> OpenTelemetry.Trace.TracerProviderBuilder! static OpenTelemetry.Trace.TracerProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Trace.TracerProviderBuilder! builder, System.Action? configure) -> OpenTelemetry.Trace.TracerProviderBuilder! diff --git a/src/OpenTelemetry.Instrumentation.Hangfire/HangfireInstrumentationOptions.cs b/src/OpenTelemetry.Instrumentation.Hangfire/HangfireInstrumentationOptions.cs index 467fe880b6..2a931b2804 100644 --- a/src/OpenTelemetry.Instrumentation.Hangfire/HangfireInstrumentationOptions.cs +++ b/src/OpenTelemetry.Instrumentation.Hangfire/HangfireInstrumentationOptions.cs @@ -45,4 +45,14 @@ public class HangfireInstrumentationOptions /// /// public Func? Filter { get; set; } + + /// + /// Gets or sets a value indicating whether to record queue latency metrics. + /// + /// + /// When enabled, records the time jobs spend waiting in the queue before execution. + /// This requires an additional database call per job execution to retrieve the enqueue timestamp. + /// Default is to avoid performance impact in high-throughput scenarios. + /// + public bool RecordQueueLatency { get; set; } } diff --git a/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetrics.cs b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetrics.cs new file mode 100644 index 0000000000..c34afbb2b3 --- /dev/null +++ b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetrics.cs @@ -0,0 +1,72 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using System.Diagnostics.Metrics; +using System.Reflection; +using OpenTelemetry.Internal; + +namespace OpenTelemetry.Instrumentation.Hangfire.Implementation; + +/// +/// Centralized metrics definitions for Hangfire instrumentation. +/// +internal static class HangfireMetrics +{ + private static readonly Assembly Assembly = typeof(HangfireMetrics).Assembly; + private static readonly AssemblyName AssemblyName = Assembly.GetName(); + internal static readonly string MeterName = AssemblyName.Name!; + private static readonly string InstrumentationVersion = Assembly.GetPackageVersion(); + + // Metric name constants + internal const string ExecutionCountMetricName = "workflow.execution.count"; + internal const string ExecutionDurationMetricName = "workflow.execution.duration"; + internal const string ExecutionStatusMetricName = "workflow.execution.status"; + internal const string ExecutionErrorsMetricName = "workflow.execution.errors"; + + internal const string QueueLatencyMetricName = "hangfire.queue.latency"; + + /// + /// The meter instance for all Hangfire metrics. + /// + public static readonly Meter Meter = new(MeterName, InstrumentationVersion); + + /// + /// Counter for the number of task executions which have been initiated. + /// Follows OpenTelemetry workflow semantic conventions. + /// + public static readonly Counter ExecutionCount = + Meter.CreateCounter(ExecutionCountMetricName, unit: "{executions}", + description: "The number of task executions which have been initiated."); + + /// + /// Histogram for duration of an execution grouped by task, type and result. + /// Follows OpenTelemetry workflow semantic conventions. + /// + public static readonly Histogram ExecutionDuration = + Meter.CreateHistogram(ExecutionDurationMetricName, unit: "s", + description: "Duration of an execution grouped by task, type and result."); + + /// + /// UpDownCounter for the number of actively running tasks grouped by task and state. + /// Follows OpenTelemetry workflow semantic conventions. + /// + public static readonly UpDownCounter ExecutionStatus = + Meter.CreateUpDownCounter(ExecutionStatusMetricName, unit: "{executions}", + description: "The number of actively running tasks grouped by task, type and the current state."); + + /// + /// Counter for the number of errors encountered in task runs (eg. compile, test failures). + /// Follows OpenTelemetry workflow semantic conventions. + /// + public static readonly Counter ExecutionErrors = + Meter.CreateCounter(ExecutionErrorsMetricName, unit: "{error}", + description: "The number of errors encountered in task runs (eg. compile, test failures)."); + + /// + /// Histogram for time tasks spend waiting in queue before execution. + /// Hangfire-specific metric (not part of standard workflow conventions). + /// + public static readonly Histogram QueueLatency = + Meter.CreateHistogram(QueueLatencyMetricName, unit: "s", + description: "Time tasks spend waiting in queue before execution starts."); +} diff --git a/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsErrorFilterAttribute.cs b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsErrorFilterAttribute.cs new file mode 100644 index 0000000000..24f76b25be --- /dev/null +++ b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsErrorFilterAttribute.cs @@ -0,0 +1,30 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using Hangfire.Common; +using Hangfire.Server; + +namespace OpenTelemetry.Instrumentation.Hangfire.Implementation; + +/// +/// Hangfire filter that records OpenTelemetry error metrics for job execution failures. +/// Follows OpenTelemetry workflow semantic conventions for workflow.execution.errors metric. +/// +internal sealed class HangfireMetricsErrorFilterAttribute : JobFilterAttribute, IServerFilter +{ + public void OnPerforming(PerformingContext performingContext) + { + } + + public void OnPerformed(PerformedContext performedContext) + { + if (performedContext.Exception != null) + { + var errorTags = HangfireTagBuilder.BuildErrorTags( + performedContext.BackgroundJob.Job, + performedContext.Exception); + + HangfireMetrics.ExecutionErrors.Add(1, errorTags); + } + } +} diff --git a/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsInstrumentation.cs b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsInstrumentation.cs new file mode 100644 index 0000000000..eba09572ee --- /dev/null +++ b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsInstrumentation.cs @@ -0,0 +1,42 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using Hangfire; +using OpenTelemetry.Trace; + +namespace OpenTelemetry.Instrumentation.Hangfire.Implementation; + +/// +/// Hangfire metrics instrumentation following OpenTelemetry workflow semantic conventions. +/// +internal sealed class HangfireMetricsInstrumentation : IDisposable +{ + private readonly List filters = new(); + + public HangfireMetricsInstrumentation(HangfireInstrumentationOptions options) + { + this.AddFilter(new HangfireMetricsJobFilterAttribute()); + this.AddFilter(new HangfireMetricsStateFilter()); + this.AddFilter(new HangfireMetricsErrorFilterAttribute()); + + // Only register queue latency filter if enabled (requires DB call per job) + if (options.RecordQueueLatency) + { + this.AddFilter(new HangfireQueueLatencyFilterAttribute()); + } + } + + public void Dispose() + { + foreach (var filter in this.filters) + { + GlobalJobFilters.Filters.Remove(filter); + } + } + + private void AddFilter(object filter) + { + this.filters.Add(filter); + GlobalJobFilters.Filters.Add(filter); + } +} diff --git a/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsJobFilterAttribute.cs b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsJobFilterAttribute.cs new file mode 100644 index 0000000000..821c71810e --- /dev/null +++ b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsJobFilterAttribute.cs @@ -0,0 +1,36 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using System.Diagnostics; +using Hangfire.Common; +using Hangfire.Server; + +namespace OpenTelemetry.Instrumentation.Hangfire.Implementation; + +/// +/// Hangfire filter that records OpenTelemetry metrics for job execution. +/// +internal sealed class HangfireMetricsJobFilterAttribute : JobFilterAttribute, IServerFilter +{ + private const string StopwatchKey = "OpenTelemetry.Metrics.Stopwatch"; + + public void OnPerforming(PerformingContext performingContext) + { + performingContext.Items[StopwatchKey] = Stopwatch.StartNew(); + } + + public void OnPerformed(PerformedContext performedContext) + { + var executionTags = HangfireTagBuilder.BuildExecutionTags(performedContext.BackgroundJob.Job, performedContext.Exception); + + HangfireMetrics.ExecutionCount.Add(1, executionTags); + + if (performedContext.Items.TryGetValue(StopwatchKey, out var stopwatchObj) && stopwatchObj is Stopwatch stopwatch) + { + stopwatch.Stop(); + var duration = stopwatch.Elapsed.TotalSeconds; + + HangfireMetrics.ExecutionDuration.Record(duration, executionTags); + } + } +} diff --git a/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsStateFilter.cs b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsStateFilter.cs new file mode 100644 index 0000000000..2576c7ee33 --- /dev/null +++ b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsStateFilter.cs @@ -0,0 +1,107 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using Hangfire.Common; +using Hangfire.States; +using Hangfire.Storage; + +namespace OpenTelemetry.Instrumentation.Hangfire.Implementation; + +/// +/// Hangfire state change filter responsible for emitting workflow execution status metrics. +/// +internal sealed class HangfireMetricsStateFilter : JobFilterAttribute, IApplyStateFilter +{ + public void OnStateApplied(ApplyStateContext context, IWriteOnlyTransaction transaction) + { + var workflowState = HangfireTagBuilder.MapWorkflowState(context.NewState.Name); + if (workflowState == null) + { + return; + } + + var errorType = GetErrorTypeFromNewState(context.NewState); + var recurringJobId = GetRecurringJobId(context); + var tags = HangfireTagBuilder.BuildStateTags( + context.BackgroundJob.Job, + workflowState, + errorType, + recurringJobId); + + HangfireMetrics.ExecutionStatus.Add(1, tags); + } + + public void OnStateUnapplied(ApplyStateContext context, IWriteOnlyTransaction transaction) + { + var workflowState = HangfireTagBuilder.MapWorkflowState(context.OldStateName); + if (workflowState == null) + { + return; + } + + var errorType = GetErrorTypeFromOldState(context); + var recurringJobId = GetRecurringJobId(context); + var tags = HangfireTagBuilder.BuildStateTags( + context.BackgroundJob.Job, + workflowState, + errorType, + recurringJobId); + + HangfireMetrics.ExecutionStatus.Add(-1, tags); + } + + private static string? GetErrorTypeFromNewState(IState state) + { + if (!string.Equals(state.Name, FailedState.StateName, StringComparison.Ordinal)) + { + return null; + } + + if (state is FailedState failedState && failedState.Exception != null) + { + var exceptionType = failedState.Exception.GetType(); + return exceptionType.FullName ?? exceptionType.Name; + } + + return TryGetExceptionTypeFromSerializedData(state.SerializeData()); + } + + private static string? GetErrorTypeFromOldState(ApplyStateContext context) + { + if (!string.Equals(context.OldStateName, FailedState.StateName, StringComparison.Ordinal)) + { + return null; + } + + StateData? stateData = context.Connection.GetStateData(context.BackgroundJob.Id); + return stateData != null ? TryGetExceptionTypeFromSerializedData(stateData.Data) : null; + } + + private static string? TryGetExceptionTypeFromSerializedData(IDictionary? data) + { + if (data == null) + { + return null; + } + + if (data.TryGetValue("ExceptionType", out var exceptionType) && !string.IsNullOrWhiteSpace(exceptionType)) + { + return exceptionType; + } + + return null; + } + + private static string? GetRecurringJobId(ApplyStateContext context) + { + try + { + return context.Connection.GetJobParameter(context.BackgroundJob.Id, "RecurringJobId"); + } + catch + { + // Parameter doesn't exist or couldn't be retrieved + return null; + } + } +} diff --git a/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireQueueLatencyFilterAttribute.cs b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireQueueLatencyFilterAttribute.cs new file mode 100644 index 0000000000..1833de18ba --- /dev/null +++ b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireQueueLatencyFilterAttribute.cs @@ -0,0 +1,71 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using Hangfire.Common; +using Hangfire.Server; +using Hangfire.States; +using DateTime = System.DateTime; + +namespace OpenTelemetry.Instrumentation.Hangfire.Implementation; + +/// +/// Hangfire filter that records queue latency metrics. +/// +/// +/// This filter captures the EnqueuedAt timestamp when a job enters the Enqueued state +/// and calculates queue latency when the job starts executing. +/// +internal sealed class HangfireQueueLatencyFilterAttribute : JobFilterAttribute, IServerFilter, IElectStateFilter +{ + private const string EnqueuedAtParameter = "OpenTelemetry.EnqueuedAt"; + + public void OnStateElection(ElectStateContext context) + { + // When a job transitions to Enqueued state, capture the EnqueuedAt timestamp + if (context.CandidateState is EnqueuedState enqueuedState) + { + try + { + var enqueuedAt = enqueuedState.EnqueuedAt; + context.Connection.SetJobParameter( + context.BackgroundJob.Id, + EnqueuedAtParameter, + JobHelper.SerializeDateTime(enqueuedAt)); + } + catch + { + // Skip storing timestamp if parameter write fails + // Instrumentation must never break Hangfire's scheduling pipeline + } + } + } + + public void OnPerforming(PerformingContext performingContext) + { + try + { + // Retrieve the EnqueuedAt timestamp that was stored when the job was enqueued + var enqueuedAtStr = performingContext.Connection.GetJobParameter( + performingContext.BackgroundJob.Id, + EnqueuedAtParameter); + + if (!string.IsNullOrEmpty(enqueuedAtStr)) + { + var enqueuedAt = JobHelper.DeserializeDateTime(enqueuedAtStr); + var queueLatency = (DateTime.UtcNow - enqueuedAt).TotalSeconds; + + var tags = HangfireTagBuilder.BuildCommonTags(performingContext.BackgroundJob.Job); + HangfireMetrics.QueueLatency.Record(queueLatency, tags); + } + } + catch + { + // Skip recording if parameter retrieval fails + } + } + + public void OnPerformed(PerformedContext performedContext) + { + // No-op: This filter only handles queue latency in OnPerforming + } +} diff --git a/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireTagBuilder.cs b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireTagBuilder.cs new file mode 100644 index 0000000000..504020a3b8 --- /dev/null +++ b/src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireTagBuilder.cs @@ -0,0 +1,201 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using System.Diagnostics; +using Hangfire.Common; +using Hangfire.Server; +using Hangfire.States; + +namespace OpenTelemetry.Instrumentation.Hangfire.Implementation; + +/// +/// Tag builder for creating standardized OpenTelemetry tag lists for Hangfire job metrics. +/// Follows OpenTelemetry workflow semantic conventions. +/// +internal static class HangfireTagBuilder +{ + // Tag name constants following OpenTelemetry workflow semantic conventions + // https://github.com/open-telemetry/semantic-conventions/blob/main/docs/workflow/workflow-metrics.md + internal const string TagWorkflowTaskName = "workflow.task.name"; + internal const string TagWorkflowExecutionOutcome = "workflow.execution.outcome"; + internal const string TagWorkflowExecutionState = "workflow.execution.state"; + internal const string TagWorkflowPlatformName = "workflow.platform.name"; + internal const string TagWorkflowTriggerType = "workflow.trigger.type"; + internal const string TagErrorType = "error.type"; + + // Outcome values per semantic conventions + internal const string OutcomeSuccess = "success"; + internal const string OutcomeFailure = "failure"; + + // State values per semantic conventions + internal const string StatePending = "pending"; + internal const string StateExecuting = "executing"; + internal const string StateCompleted = "completed"; + + // Platform name constant + internal const string PlatformHangfire = "hangfire"; + + // Trigger type constants + internal const string TriggerTypeCron = "cron"; + internal const string TriggerTypeApi = "api"; + + /// + /// Creates a tag list with common job metadata following workflow semantic conventions. + /// Includes required workflow.task.name and recommended workflow.platform.name. + /// Also includes custom Hangfire-specific attributes (job.type, job.method). + /// + /// The Hangfire job. + /// Tag list with common job tags. + public static TagList BuildCommonTags(Job job) + { + var tags = new TagList + { + GetTaskName(job), + GetPlatformName(), + }; + return tags; + } + + /// + /// Creates a tag list representing workflow execution state transitions. + /// Includes required workflow.task.name, workflow.execution.state, and workflow.trigger.type tags, + /// recommended workflow.platform.name, and conditionally required error.type. + /// + /// The Hangfire job. + /// The workflow state value. + /// Optional error type to annotate failure states. + /// Optional recurring job ID if this job was triggered by a recurring job. + /// Tag list suitable for workflow.execution.status metric. + public static TagList BuildStateTags(Job job, string workflowState, string? errorType, string? recurringJobId) + { + var tags = new TagList + { + GetTaskName(job), + GetPlatformName(), + GetState(workflowState), + GetTriggerType(recurringJobId), + }; + + if (!string.IsNullOrEmpty(errorType)) + { + tags.Add(new KeyValuePair(TagErrorType, errorType)); + } + + return tags; + } + + /// + /// Maps Hangfire state names to workflow semantic convention state values. + /// + /// Hangfire state name. + /// Mapped workflow state value, or if the state is not recognized. + public static string? MapWorkflowState(string? hangfireState) + { + if (hangfireState == ScheduledState.StateName || + hangfireState == EnqueuedState.StateName || + hangfireState == AwaitingState.StateName) + { + return StatePending; + } + + if (hangfireState == ProcessingState.StateName) + { + return StateExecuting; + } + + if (hangfireState == SucceededState.StateName || + hangfireState == DeletedState.StateName || + hangfireState == FailedState.StateName) + { + return StateCompleted; + } + + return null; + } + + /// + /// Creates a tag list with execution result tags following workflow semantic conventions. + /// Includes required attributes (workflow.task.name, workflow.execution.outcome), + /// recommended attributes (workflow.platform.name), + /// conditionally required attributes (error.type if failed) + /// + /// The Hangfire job. + /// The exception, if any occurred. + /// Tag list with execution result tags. + public static TagList BuildExecutionTags(Job job, Exception? exception) + { + var tags = new TagList + { + GetTaskName(job), + GetPlatformName(), + GetOutcome(exception), + }; + + // Conditionally Required: error.type (if and only if the task run failed) + if (exception is not null) + { + tags.Add(GetErrorType(exception)); + } + + return tags; + } + + /// + /// Creates a tag list for workflow.execution.errors metric following workflow semantic conventions. + /// Includes required attributes (error.type, workflow.task.name), + /// recommended attributes (workflow.platform.name). + /// + /// The Hangfire job. + /// The exception that occurred. + /// Tag list with error tags. + public static TagList BuildErrorTags(Job job, Exception exception) + { + var tags = new TagList + { + GetErrorType(exception), + GetTaskName(job), + GetPlatformName(), + }; + + return tags; + } + + // Required workflow attributes + private static KeyValuePair GetTaskName(Job job) => + new(TagWorkflowTaskName, job.ToString() ?? "unknown"); + + private static KeyValuePair GetOutcome(Exception? exception) => + new(TagWorkflowExecutionOutcome, exception is null ? OutcomeSuccess : OutcomeFailure); + + private static KeyValuePair GetTriggerType(string? recurringJobId) + { + // Check if job was triggered by a recurring job (cron) + if (!string.IsNullOrEmpty(recurringJobId)) + { + return new(TagWorkflowTriggerType, TriggerTypeCron); + } + + // Default to API trigger (fire-and-forget, scheduled, continuations) + return new(TagWorkflowTriggerType, TriggerTypeApi); + } + + // Recommended workflow attributes + private static KeyValuePair GetPlatformName() => + new(TagWorkflowPlatformName, PlatformHangfire); + + private static KeyValuePair GetState(string workflowState) => + new(TagWorkflowExecutionState, workflowState); + + // Conditionally required workflow attributes + private static KeyValuePair GetErrorType(Exception exception) + { + var loggedException = exception; + + if (loggedException is JobPerformanceException pe) + { + loggedException = pe.InnerException; + } + + return new KeyValuePair(TagErrorType, loggedException.GetType().FullName ?? loggedException.GetType().Name); + } +} diff --git a/src/OpenTelemetry.Instrumentation.Hangfire/MeterProviderBuilderExtensions.cs b/src/OpenTelemetry.Instrumentation.Hangfire/MeterProviderBuilderExtensions.cs new file mode 100644 index 0000000000..efb629a493 --- /dev/null +++ b/src/OpenTelemetry.Instrumentation.Hangfire/MeterProviderBuilderExtensions.cs @@ -0,0 +1,66 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using OpenTelemetry.Instrumentation.Hangfire.Implementation; +using OpenTelemetry.Internal; +using OpenTelemetry.Trace; + +namespace OpenTelemetry.Metrics; + +/// +/// Extension methods to simplify registering of Hangfire metrics instrumentation. +/// +public static class MeterProviderBuilderExtensions +{ + /// + /// Enables Hangfire metrics instrumentation. + /// + /// being configured. + /// The instance of to chain the calls. + public static MeterProviderBuilder AddHangfireInstrumentation( + this MeterProviderBuilder builder) + => AddHangfireInstrumentation(builder, name: null, configure: null); + + /// + /// Enables Hangfire metrics instrumentation. + /// + /// being configured. + /// Callback action for configuring . + /// The instance of to chain the calls. + public static MeterProviderBuilder AddHangfireInstrumentation( + this MeterProviderBuilder builder, + Action? configure) + => AddHangfireInstrumentation(builder, name: null, configure); + + /// + /// Enables Hangfire metrics instrumentation. + /// + /// being configured. + /// Name which is used when retrieving options. + /// configuration options. + /// The instance of to chain the calls. + public static MeterProviderBuilder AddHangfireInstrumentation( + this MeterProviderBuilder builder, + string? name, + Action? configure) + { + Guard.ThrowIfNull(builder); + + name ??= Options.DefaultName; + + if (configure != null) + { + builder.ConfigureServices(services => services.Configure(name, configure)); + } + + builder.AddMeter(HangfireMetrics.MeterName); + + return builder.AddInstrumentation(sp => + { + var options = sp.GetRequiredService>().Get(name); + return new HangfireMetricsInstrumentation(options); + }); + } +} diff --git a/src/OpenTelemetry.Instrumentation.Hangfire/OpenTelemetry.Instrumentation.Hangfire.csproj b/src/OpenTelemetry.Instrumentation.Hangfire/OpenTelemetry.Instrumentation.Hangfire.csproj index 98e454f467..1f3d8222e6 100644 --- a/src/OpenTelemetry.Instrumentation.Hangfire/OpenTelemetry.Instrumentation.Hangfire.csproj +++ b/src/OpenTelemetry.Instrumentation.Hangfire/OpenTelemetry.Instrumentation.Hangfire.csproj @@ -5,7 +5,6 @@ OpenTelemetry Hangfire Instrumentation. $(PackageTags);Hangfire Instrumentation.Hangfire- - false