From d5466a25db20d180ceb600b1457a2da7cb65fa84 Mon Sep 17 00:00:00 2001 From: danielmarbach Date: Tue, 1 Oct 2024 18:38:01 +0200 Subject: [PATCH 1/7] Use locks to serialize arm/disarm actions --- .../RepeatedFailuresOverTimeCircuitBreaker.cs | 86 +++++++++++++++---- 1 file changed, 69 insertions(+), 17 deletions(-) diff --git a/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs b/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs index 80ae22d4..360b454a 100644 --- a/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs +++ b/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs @@ -5,53 +5,103 @@ using System.Threading.Tasks; using Logging; + /// + /// A circuit breaker that is armed on a failure and disarmed on success. After in the + /// armed state, the will fire. The and allow + /// changing other state when the circuit breaker is armed or disarmed. + /// sealed class RepeatedFailuresOverTimeCircuitBreaker { + /// + /// A circuit breaker that is armed on a failure and disarmed on success. After in the + /// armed state, the will fire. The and allow + /// changing other state when the circuit breaker is armed or disarmed. + /// + /// A name that is output in log messages when the circuit breaker changes states. + /// The time to wait after the first failure before triggering. + /// The action to take when the circuit breaker is triggered. + /// The action to execute on the first failure. + /// WARNING: This action is called from within a lock to serialize arming and disarming actions. + /// The action to execute when a success disarms the circuit breaker. + /// WARNING: This action is called from within a lock to serialize arming and disarming actions. + /// How long to delay on each failure when in the Triggered state. Defaults to 10 seconds. + /// How long to delay on each failure when in the Armed state. Defaults to 1 second. public RepeatedFailuresOverTimeCircuitBreaker(string name, TimeSpan timeToWaitBeforeTriggering, Action triggerAction, - Action armedAction, - Action disarmedAction) + Action armedAction = null, + Action disarmedAction = null, + TimeSpan? timeToWaitWhenTriggered = default, + TimeSpan? timeToWaitWhenArmed = default) { this.name = name; this.triggerAction = triggerAction; this.armedAction = armedAction; this.disarmedAction = disarmedAction; this.timeToWaitBeforeTriggering = timeToWaitBeforeTriggering; + this.timeToWaitWhenTriggered = timeToWaitWhenTriggered ?? TimeSpan.FromSeconds(10); + this.timeToWaitWhenArmed = timeToWaitWhenArmed ?? TimeSpan.FromSeconds(1); timer = new Timer(CircuitBreakerTriggered); } + /// + /// Log a success, disarming the circuit breaker if it was previously armed. + /// public void Success() { - var previousState = Interlocked.CompareExchange(ref circuitBreakerState, Disarmed, Armed); - - // If the circuit breaker was Armed or triggered before, disarm it - if (previousState == Armed || Interlocked.CompareExchange(ref circuitBreakerState, Disarmed, Triggered) == Triggered) + // Check the status of the circuit breaker, exiting early outside the lock if already disarmed + var previousState = circuitBreakerState; + if (previousState != Disarmed) { - _ = timer.Change(Timeout.Infinite, Timeout.Infinite); - Logger.InfoFormat("The circuit breaker for {0} is now disarmed", name); - disarmedAction(); + lock (timer) + { + // Recheck state after obtaining the lock + previousState = circuitBreakerState; + if (previousState != Disarmed) + { + circuitBreakerState = Disarmed; + _ = timer.Change(Timeout.Infinite, Timeout.Infinite); + Logger.InfoFormat("The circuit breaker for {0} is now disarmed", name); + disarmedAction?.Invoke(); + } + } } } + /// + /// Log a failure, arming the circuit breaker if it was previously disarmed. + /// + /// The exception that caused the failure. + /// A cancellation token. public Task Failure(Exception exception, CancellationToken cancellationToken = default) { + // Atomically store the exception that caused the circuit breaker to trip _ = Interlocked.Exchange(ref lastException, exception); - // Atomically set state to Armed if it was previously Disarmed - var previousState = Interlocked.CompareExchange(ref circuitBreakerState, Armed, Disarmed); - + // Check the status of the circuit breaker, exiting early outside the lock if already armed or triggered + var previousState = circuitBreakerState; if (previousState == Disarmed) { - armedAction(); - _ = timer.Change(timeToWaitBeforeTriggering, NoPeriodicTriggering); - Logger.WarnFormat("The circuit breaker for {0} is now in the armed state due to {1}", name, exception); + lock (timer) + { + // Recheck state after obtaining the lock + previousState = circuitBreakerState; + if (previousState == Disarmed) + { + circuitBreakerState = Armed; + armedAction?.Invoke(); + _ = timer.Change(timeToWaitBeforeTriggering, NoPeriodicTriggering); + Logger.WarnFormat("The circuit breaker for {0} is now in the armed state due to {1}", name, exception); + } + } } - // If the circuit breaker has been triggered, wait for 10 seconds before proceeding to prevent flooding the logs and hammering the ServiceBus - return Task.Delay(previousState == Triggered ? TimeSpan.FromSeconds(10) : TimeSpan.FromSeconds(1), cancellationToken); + return Task.Delay(previousState == Triggered ? timeToWaitWhenTriggered : timeToWaitWhenArmed, cancellationToken); } + /// + /// Disposes the resources associated with the circuit breaker. + /// public void Dispose() => timer?.Dispose(); void CircuitBreakerTriggered(object state) @@ -74,6 +124,8 @@ void CircuitBreakerTriggered(object state) readonly Action triggerAction; readonly Action armedAction; readonly Action disarmedAction; + readonly TimeSpan timeToWaitWhenTriggered; + readonly TimeSpan timeToWaitWhenArmed; const int Disarmed = 0; const int Armed = 1; From eca904be012a08e2aaa27e61da65cdede03a48de Mon Sep 17 00:00:00 2001 From: Daniel Marbach Date: Wed, 2 Oct 2024 00:14:48 +0200 Subject: [PATCH 2/7] Explicit state lock, Volatile read of state outside lock, actually reduce nesting where possible --- .../RepeatedFailuresOverTimeCircuitBreaker.cs | 94 ++++++++++++------- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs b/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs index 360b454a..4d82bc2c 100644 --- a/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs +++ b/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs @@ -1,4 +1,6 @@ -namespace NServiceBus.Transport.AzureServiceBus +#nullable enable + +namespace NServiceBus.Transport.AzureServiceBus { using System; using System.Threading; @@ -28,15 +30,15 @@ sealed class RepeatedFailuresOverTimeCircuitBreaker /// How long to delay on each failure when in the Armed state. Defaults to 1 second. public RepeatedFailuresOverTimeCircuitBreaker(string name, TimeSpan timeToWaitBeforeTriggering, Action triggerAction, - Action armedAction = null, - Action disarmedAction = null, + Action? armedAction = null, + Action? disarmedAction = null, TimeSpan? timeToWaitWhenTriggered = default, TimeSpan? timeToWaitWhenArmed = default) { this.name = name; this.triggerAction = triggerAction; - this.armedAction = armedAction; - this.disarmedAction = disarmedAction; + this.armedAction = armedAction ?? (static () => { }); + this.disarmedAction = disarmedAction ?? (static () => { }); this.timeToWaitBeforeTriggering = timeToWaitBeforeTriggering; this.timeToWaitWhenTriggered = timeToWaitWhenTriggered ?? TimeSpan.FromSeconds(10); this.timeToWaitWhenArmed = timeToWaitWhenArmed ?? TimeSpan.FromSeconds(1); @@ -50,21 +52,23 @@ public RepeatedFailuresOverTimeCircuitBreaker(string name, TimeSpan timeToWaitBe public void Success() { // Check the status of the circuit breaker, exiting early outside the lock if already disarmed - var previousState = circuitBreakerState; - if (previousState != Disarmed) + if (Volatile.Read(ref circuitBreakerState) == Disarmed) + { + return; + } + + lock (stateLock) { - lock (timer) + // Recheck state after obtaining the lock + if (circuitBreakerState == Disarmed) { - // Recheck state after obtaining the lock - previousState = circuitBreakerState; - if (previousState != Disarmed) - { - circuitBreakerState = Disarmed; - _ = timer.Change(Timeout.Infinite, Timeout.Infinite); - Logger.InfoFormat("The circuit breaker for {0} is now disarmed", name); - disarmedAction?.Invoke(); - } + return; } + + circuitBreakerState = Disarmed; + _ = timer.Change(Timeout.Infinite, Timeout.Infinite); + Logger.InfoFormat("The circuit breaker for {0} is now disarmed", name); + disarmedAction(); } } @@ -79,44 +83,61 @@ public Task Failure(Exception exception, CancellationToken cancellationToken = d _ = Interlocked.Exchange(ref lastException, exception); // Check the status of the circuit breaker, exiting early outside the lock if already armed or triggered - var previousState = circuitBreakerState; - if (previousState == Disarmed) + var previousState = Volatile.Read(ref circuitBreakerState); + if (previousState != Disarmed) { - lock (timer) + return Delay(); + } + + lock (stateLock) + { + // Recheck state after obtaining the lock + previousState = circuitBreakerState; + if (previousState != Disarmed) { - // Recheck state after obtaining the lock - previousState = circuitBreakerState; - if (previousState == Disarmed) - { - circuitBreakerState = Armed; - armedAction?.Invoke(); - _ = timer.Change(timeToWaitBeforeTriggering, NoPeriodicTriggering); - Logger.WarnFormat("The circuit breaker for {0} is now in the armed state due to {1}", name, exception); - } + return Delay(); } + + circuitBreakerState = Armed; + armedAction(); + _ = timer.Change(timeToWaitBeforeTriggering, NoPeriodicTriggering); + Logger.WarnFormat("The circuit breaker for {0} is now in the armed state due to {1}", name, exception); } - return Task.Delay(previousState == Triggered ? timeToWaitWhenTriggered : timeToWaitWhenArmed, cancellationToken); + return Delay(); + + Task Delay() => Task.Delay(previousState == Triggered ? timeToWaitWhenTriggered : timeToWaitWhenArmed, cancellationToken); } /// /// Disposes the resources associated with the circuit breaker. /// - public void Dispose() => timer?.Dispose(); + public void Dispose() => timer.Dispose(); - void CircuitBreakerTriggered(object state) + void CircuitBreakerTriggered(object? state) { - if (Interlocked.CompareExchange(ref circuitBreakerState, Triggered, Armed) != Armed) + var previousState = Volatile.Read(ref circuitBreakerState); + if (previousState == Disarmed) { return; } - Logger.WarnFormat("The circuit breaker for {0} will now be triggered with exception {1}", name, lastException); - triggerAction(lastException); + lock (stateLock) + { + if (circuitBreakerState == Disarmed) + { + return; + } + + circuitBreakerState = Triggered; + Logger.WarnFormat("The circuit breaker for {0} will now be triggered with exception {1}", name, lastException); + triggerAction(lastException!); + + } } int circuitBreakerState = Disarmed; - Exception lastException; + Exception? lastException; readonly string name; readonly Timer timer; @@ -126,6 +147,7 @@ void CircuitBreakerTriggered(object state) readonly Action disarmedAction; readonly TimeSpan timeToWaitWhenTriggered; readonly TimeSpan timeToWaitWhenArmed; + readonly object stateLock = new(); const int Disarmed = 0; const int Armed = 1; From 0bc4d889313733f459c674dc69412509fb13ecbb Mon Sep 17 00:00:00 2001 From: danielmarbach Date: Wed, 2 Oct 2024 11:03:30 +0200 Subject: [PATCH 3/7] Small cosmetics for better readability --- .../RepeatedFailuresOverTimeCircuitBreaker.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs b/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs index 4d82bc2c..dc6ef01b 100644 --- a/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs +++ b/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs @@ -66,6 +66,7 @@ public void Success() } circuitBreakerState = Disarmed; + _ = timer.Change(Timeout.Infinite, Timeout.Infinite); Logger.InfoFormat("The circuit breaker for {0} is now disarmed", name); disarmedAction(); @@ -82,9 +83,8 @@ public Task Failure(Exception exception, CancellationToken cancellationToken = d // Atomically store the exception that caused the circuit breaker to trip _ = Interlocked.Exchange(ref lastException, exception); - // Check the status of the circuit breaker, exiting early outside the lock if already armed or triggered var previousState = Volatile.Read(ref circuitBreakerState); - if (previousState != Disarmed) + if (previousState is Armed or Triggered) { return Delay(); } @@ -93,12 +93,15 @@ public Task Failure(Exception exception, CancellationToken cancellationToken = d { // Recheck state after obtaining the lock previousState = circuitBreakerState; - if (previousState != Disarmed) + if (previousState is Armed or Triggered) { return Delay(); } circuitBreakerState = Armed; + + // Executing the action first before starting the timer to ensure that the action is executed before the timer fires + // and the time of the action is not included in the time to wait before triggering. armedAction(); _ = timer.Change(timeToWaitBeforeTriggering, NoPeriodicTriggering); Logger.WarnFormat("The circuit breaker for {0} is now in the armed state due to {1}", name, exception); @@ -124,6 +127,7 @@ void CircuitBreakerTriggered(object? state) lock (stateLock) { + // Recheck state after obtaining the lock if (circuitBreakerState == Disarmed) { return; @@ -132,7 +136,6 @@ void CircuitBreakerTriggered(object? state) circuitBreakerState = Triggered; Logger.WarnFormat("The circuit breaker for {0} will now be triggered with exception {1}", name, lastException); triggerAction(lastException!); - } } From 896dc64b2be5d225f1b8ecba40818f984fa729fa Mon Sep 17 00:00:00 2001 From: danielmarbach Date: Wed, 2 Oct 2024 11:12:09 +0200 Subject: [PATCH 4/7] Better logging --- .../RepeatedFailuresOverTimeCircuitBreaker.cs | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs b/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs index dc6ef01b..f7ccad16 100644 --- a/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs +++ b/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs @@ -69,7 +69,15 @@ public void Success() _ = timer.Change(Timeout.Infinite, Timeout.Infinite); Logger.InfoFormat("The circuit breaker for {0} is now disarmed", name); - disarmedAction(); + try + { + disarmedAction(); + } + catch (Exception ex) + { + Logger.Error($"The circuit breaker for {name} was unable to execute the disarm action", ex); + throw; + } } } @@ -100,9 +108,18 @@ public Task Failure(Exception exception, CancellationToken cancellationToken = d circuitBreakerState = Armed; - // Executing the action first before starting the timer to ensure that the action is executed before the timer fires - // and the time of the action is not included in the time to wait before triggering. - armedAction(); + try + { + // Executing the action first before starting the timer to ensure that the action is executed before the timer fires + // and the time of the action is not included in the time to wait before triggering. + armedAction(); + } + catch (Exception ex) + { + Logger.Error($"The circuit breaker for {name} was unable to execute the arm action", new AggregateException(ex, exception)); + throw; + } + _ = timer.Change(timeToWaitBeforeTriggering, NoPeriodicTriggering); Logger.WarnFormat("The circuit breaker for {0} is now in the armed state due to {1}", name, exception); } @@ -135,7 +152,15 @@ void CircuitBreakerTriggered(object? state) circuitBreakerState = Triggered; Logger.WarnFormat("The circuit breaker for {0} will now be triggered with exception {1}", name, lastException); - triggerAction(lastException!); + + try + { + triggerAction(lastException!); + } + catch (Exception ex) + { + Logger.Fatal($"The circuit breaker for {name} was unable to execute the trigger action", new AggregateException(ex, lastException!)); + } } } From 60b55614616e9d233443b1f69fd4df17388dccec Mon Sep 17 00:00:00 2001 From: danielmarbach Date: Wed, 2 Oct 2024 11:20:17 +0200 Subject: [PATCH 5/7] Verbose comment --- .../RepeatedFailuresOverTimeCircuitBreaker.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs b/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs index f7ccad16..3870c257 100644 --- a/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs +++ b/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs @@ -23,11 +23,23 @@ sealed class RepeatedFailuresOverTimeCircuitBreaker /// The time to wait after the first failure before triggering. /// The action to take when the circuit breaker is triggered. /// The action to execute on the first failure. - /// WARNING: This action is called from within a lock to serialize arming and disarming actions. + /// Warning: This action is also invoked from within a lock. Any long-running, blocking, or I/O-bound code should be avoided + /// within this action, as it can prevent other threads from proceeding, potentially leading to contention or performance bottlenecks. + /// /// The action to execute when a success disarms the circuit breaker. - /// WARNING: This action is called from within a lock to serialize arming and disarming actions. + /// Warning: This action is also invoked from within a lock. Any long-running, blocking, or I/O-bound code should be avoided + /// within this action, as it can prevent other threads from proceeding, potentially leading to contention or performance bottlenecks. + /// /// How long to delay on each failure when in the Triggered state. Defaults to 10 seconds. /// How long to delay on each failure when in the Armed state. Defaults to 1 second. + /// + /// The and are invoked from within a lock to ensure that arming and disarming + /// actions are serialized and do not execute concurrently. As a result, care must be taken to ensure that these actions do not + /// introduce delays or deadlocks by performing lengthy operations or synchronously waiting on external resources. + /// + /// Best practice: If the logic inside these actions involves blocking or long-running tasks, consider offloading + /// the work to a background task or thread that doesn't hold the lock. + /// public RepeatedFailuresOverTimeCircuitBreaker(string name, TimeSpan timeToWaitBeforeTriggering, Action triggerAction, Action? armedAction = null, From a07821870aeb9ccfe6a23152e5a0f42b22f1437b Mon Sep 17 00:00:00 2001 From: danielmarbach Date: Wed, 2 Oct 2024 11:29:02 +0200 Subject: [PATCH 6/7] Even better logging --- .../RepeatedFailuresOverTimeCircuitBreaker.cs | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs b/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs index 3870c257..d2c4c80c 100644 --- a/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs +++ b/src/Transport/Receiving/RepeatedFailuresOverTimeCircuitBreaker.cs @@ -40,7 +40,9 @@ sealed class RepeatedFailuresOverTimeCircuitBreaker /// Best practice: If the logic inside these actions involves blocking or long-running tasks, consider offloading /// the work to a background task or thread that doesn't hold the lock. /// - public RepeatedFailuresOverTimeCircuitBreaker(string name, TimeSpan timeToWaitBeforeTriggering, + public RepeatedFailuresOverTimeCircuitBreaker( + string name, + TimeSpan timeToWaitBeforeTriggering, Action triggerAction, Action? armedAction = null, Action? disarmedAction = null, @@ -80,14 +82,14 @@ public void Success() circuitBreakerState = Disarmed; _ = timer.Change(Timeout.Infinite, Timeout.Infinite); - Logger.InfoFormat("The circuit breaker for {0} is now disarmed", name); + Logger.InfoFormat("The circuit breaker for '{0}' is now disarmed.", name); try { disarmedAction(); } catch (Exception ex) { - Logger.Error($"The circuit breaker for {name} was unable to execute the disarm action", ex); + Logger.Error($"The circuit breaker for '{name}' was unable to execute the disarm action.", ex); throw; } } @@ -128,17 +130,25 @@ public Task Failure(Exception exception, CancellationToken cancellationToken = d } catch (Exception ex) { - Logger.Error($"The circuit breaker for {name} was unable to execute the arm action", new AggregateException(ex, exception)); + Logger.Error($"The circuit breaker for '{name}' was unable to execute the arm action.", new AggregateException(ex, exception)); throw; } _ = timer.Change(timeToWaitBeforeTriggering, NoPeriodicTriggering); - Logger.WarnFormat("The circuit breaker for {0} is now in the armed state due to {1}", name, exception); + Logger.WarnFormat("The circuit breaker for '{0}' is now in the armed state due to '{1}' and might trigger in '{2}' when not disarmed.", name, exception, timeToWaitBeforeTriggering); } return Delay(); - Task Delay() => Task.Delay(previousState == Triggered ? timeToWaitWhenTriggered : timeToWaitWhenArmed, cancellationToken); + Task Delay() + { + var timeToWait = previousState == Triggered ? timeToWaitWhenTriggered : timeToWaitWhenArmed; + if (Logger.IsDebugEnabled) + { + Logger.DebugFormat("The circuit breaker for '{0}' is delaying the operation by '{1}'.", name, timeToWait); + } + return Task.Delay(timeToWait, cancellationToken); + } } /// @@ -163,7 +173,7 @@ void CircuitBreakerTriggered(object? state) } circuitBreakerState = Triggered; - Logger.WarnFormat("The circuit breaker for {0} will now be triggered with exception {1}", name, lastException); + Logger.WarnFormat("The circuit breaker for '{0}' will now be triggered with exception '{1}'.", name, lastException); try { @@ -171,7 +181,7 @@ void CircuitBreakerTriggered(object? state) } catch (Exception ex) { - Logger.Fatal($"The circuit breaker for {name} was unable to execute the trigger action", new AggregateException(ex, lastException!)); + Logger.Fatal($"The circuit breaker for '{name}' was unable to execute the trigger action.", new AggregateException(ex, lastException!)); } } } From 1232e0a300d666f31a4a7d8bdc9e84d65170f7fe Mon Sep 17 00:00:00 2001 From: danielmarbach Date: Wed, 2 Oct 2024 12:19:43 +0200 Subject: [PATCH 7/7] Basic test coverage --- ...atedFailuresOverTimeCircuitBreakerTests.cs | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 src/Tests/Receiving/RepeatedFailuresOverTimeCircuitBreakerTests.cs diff --git a/src/Tests/Receiving/RepeatedFailuresOverTimeCircuitBreakerTests.cs b/src/Tests/Receiving/RepeatedFailuresOverTimeCircuitBreakerTests.cs new file mode 100644 index 00000000..9f2fc77b --- /dev/null +++ b/src/Tests/Receiving/RepeatedFailuresOverTimeCircuitBreakerTests.cs @@ -0,0 +1,222 @@ +namespace NServiceBus.Transport.AzureServiceBus.Tests.Receiving +{ + using System; + using System.Diagnostics; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using NUnit.Framework; + + // Ideally the circuit breaker would use a time provider to allow for easier testing but that would require a significant refactor + // and we want keep the changes to a minimum for now to allow backporting to older versions. + [TestFixture] + public class RepeatedFailuresOverTimeCircuitBreakerTests + { + [Test] + public async Task Should_disarm_on_success() + { + var armedActionCalled = false; + var disarmedActionCalled = false; + + var circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker( + "TestCircuitBreaker", + TimeSpan.FromMilliseconds(100), + ex => { }, + () => armedActionCalled = true, + () => disarmedActionCalled = true, + TimeSpan.Zero, + TimeSpan.Zero + ); + + await circuitBreaker.Failure(new Exception("Test Exception")); + circuitBreaker.Success(); + + Assert.That(armedActionCalled, Is.True, "The armed action should be called."); + Assert.That(disarmedActionCalled, Is.True, "The disarmed action should be called."); + } + + [Test] + public async Task Should_rethrow_exception_on_success() + { + var circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker( + "TestCircuitBreaker", + TimeSpan.FromMilliseconds(100), + ex => { }, + () => { }, + () => throw new Exception("Exception from disarmed action"), + timeToWaitWhenTriggered: TimeSpan.Zero, + timeToWaitWhenArmed: TimeSpan.Zero + ); + + await circuitBreaker.Failure(new Exception("Test Exception")); + + var ex = Assert.Throws(() => circuitBreaker.Success()); + Assert.That(ex.Message, Is.EqualTo("Exception from disarmed action")); + } + + [Test] + public async Task Should_trigger_after_failure_timeout() + { + var triggerActionCalled = false; + Exception lastTriggerException = null; + + var circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker( + "TestCircuitBreaker", + TimeSpan.Zero, + ex => { triggerActionCalled = true; lastTriggerException = ex; }, + timeToWaitWhenTriggered: TimeSpan.Zero, + timeToWaitWhenArmed: TimeSpan.FromMilliseconds(100) + ); + + await circuitBreaker.Failure(new Exception("Test Exception")); + + Assert.That(triggerActionCalled, Is.True, "The trigger action should be called after timeout."); + Assert.That(lastTriggerException, Is.Not.Null, "The exception passed to the trigger action should not be null."); + } + + [Test] + public void Should_rethrow_exception_on_failure() + { + var circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker( + "TestCircuitBreaker", + TimeSpan.FromMilliseconds(100), + ex => { }, + () => throw new Exception("Exception from armed action"), + () => { }, + timeToWaitWhenTriggered: TimeSpan.Zero, + timeToWaitWhenArmed: TimeSpan.Zero + ); + + var ex = Assert.ThrowsAsync(async () => await circuitBreaker.Failure(new Exception("Test Exception"))); + Assert.That(ex.Message, Is.EqualTo("Exception from armed action")); + } + + [Test] + public async Task Should_delay_after_trigger_failure() + { + var timeToWaitWhenTriggered = TimeSpan.FromMilliseconds(50); + var timeToWaitWhenArmed = TimeSpan.FromMilliseconds(100); + + var circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker( + "TestCircuitBreaker", + TimeSpan.Zero, + _ => { }, + timeToWaitWhenTriggered: timeToWaitWhenTriggered, + timeToWaitWhenArmed: timeToWaitWhenArmed + ); + + var stopWatch = Stopwatch.StartNew(); + + await circuitBreaker.Failure(new Exception("Test Exception")); + await circuitBreaker.Failure(new Exception("Test Exception After Trigger")); + + stopWatch.Stop(); + + Assert.That(stopWatch.ElapsedMilliseconds, Is.GreaterThanOrEqualTo(timeToWaitWhenTriggered.Add(timeToWaitWhenArmed).TotalMilliseconds).Within(20), "The circuit breaker should delay after a triggered failure."); + } + + [Test] + public async Task Should_not_trigger_if_disarmed_before_timeout() + { + var triggerActionCalled = false; + + var circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker( + "TestCircuitBreaker", + TimeSpan.FromMilliseconds(100), + ex => triggerActionCalled = true, + timeToWaitWhenTriggered: TimeSpan.Zero, + timeToWaitWhenArmed: TimeSpan.Zero + ); + + await circuitBreaker.Failure(new Exception("Test Exception")); + circuitBreaker.Success(); + + Assert.That(triggerActionCalled, Is.False, "The trigger action should not be called if the circuit breaker was disarmed."); + } + + [Test] + public async Task Should_handle_concurrent_failure_and_success() + { + var armedActionCalled = false; + var disarmedActionCalled = false; + var triggerActionCalled = false; + + var circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker( + "TestCircuitBreaker", + TimeSpan.FromMilliseconds(100), + ex => triggerActionCalled = true, + () => armedActionCalled = true, + () => disarmedActionCalled = true, + TimeSpan.Zero, + TimeSpan.Zero + ); + + var failureTask = circuitBreaker.Failure(new Exception("Test Exception")); + var successTask = Task.Run(() => + { + Thread.Sleep(50); // Simulate some delay before success + circuitBreaker.Success(); + }); + + await Task.WhenAll(failureTask, successTask); + + Assert.That(armedActionCalled, Is.True, "The armed action should be called."); + Assert.That(disarmedActionCalled, Is.True, "The disarmed action should be called."); + Assert.That(triggerActionCalled, Is.False, "The trigger action should not be called if success occurred before timeout."); + } + + [Test] + public async Task Should_handle_high_concurrent_failure_and_success() + { + var armedActionCalled = 0; + var disarmedActionCalled = 0; + var triggerActionCalled = 0; + + var circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker( + "TestCircuitBreaker", + TimeSpan.FromSeconds(5), + ex => Interlocked.Increment(ref triggerActionCalled), + () => Interlocked.Increment(ref armedActionCalled), + () => Interlocked.Increment(ref disarmedActionCalled), + TimeSpan.Zero, + TimeSpan.FromMilliseconds(25) + ); + + var tasks = Enumerable.Range(0, 1000) + .Select( + i => i % 2 == 0 ? + circuitBreaker.Failure(new Exception($"Test Exception {i}")) : + Task.Run(() => + { + Thread.Sleep(25); // Simulate some delay before success + circuitBreaker.Success(); + }) + ).ToArray(); + + await Task.WhenAll(tasks); + + Assert.That(armedActionCalled, Is.EqualTo(1), "The armed action should be called."); + Assert.That(disarmedActionCalled, Is.EqualTo(1), "The disarmed action should be called."); + Assert.That(triggerActionCalled, Is.Zero, "The trigger action should not be called if success occurred before timeout."); + } + + [Test] + public async Task Should_trigger_after_multiple_failures_and_timeout() + { + var triggerActionCalled = false; + + var circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker( + "TestCircuitBreaker", + TimeSpan.FromMilliseconds(50), + ex => triggerActionCalled = true, + timeToWaitWhenTriggered: TimeSpan.FromMilliseconds(50), + timeToWaitWhenArmed: TimeSpan.FromMilliseconds(50) + ); + + await circuitBreaker.Failure(new Exception("Test Exception")); + await circuitBreaker.Failure(new Exception("Another Exception After Trigger")); + + Assert.That(triggerActionCalled, Is.True, "The trigger action should be called after repeated failures and timeout."); + } + } +} \ No newline at end of file