diff --git a/.editorconfig b/.editorconfig index 819a41fe1e..4590621c96 100644 --- a/.editorconfig +++ b/.editorconfig @@ -198,3 +198,9 @@ dotnet_diagnostic.CA1416.severity = silent dotnet_code_quality.CA2100.excluded_type_names_with_derived_types = Microsoft.Data.SqlClient.ManualTesting.Tests.* dotnet_diagnostic.xUnit1031.severity=none dotnet_diagnostic.xUnit1030.severity=none + +# Disables warning for unnamed enum case values +# e.g. providing an invalid int value for an enum that wraps int +dotnet_diagnostic.CS8524.severity = none +# Treat missing enum cases as errors +dotnet_diagnostic.CS8509.severity = error \ No newline at end of file diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs index 568592072d..bee990bdc1 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs @@ -149,9 +149,10 @@ internal static Exception ExceptionWithStackTrace(Exception e) } } +#nullable enable internal static Timer UnsafeCreateTimer( TimerCallback callback, - object state, + object? state, int dueTimeMilliseconds, int periodMilliseconds) => UnsafeCreateTimer( @@ -160,7 +161,7 @@ internal static Timer UnsafeCreateTimer( TimeSpan.FromMilliseconds(dueTimeMilliseconds), TimeSpan.FromMilliseconds(periodMilliseconds)); - internal static Timer UnsafeCreateTimer(TimerCallback callback, object state, TimeSpan dueTime, TimeSpan period) + internal static Timer UnsafeCreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) { // Don't capture the current ExecutionContext and its AsyncLocals onto // a global timer causing them to live forever @@ -175,6 +176,39 @@ internal static Timer UnsafeCreateTimer(TimerCallback callback, object state, Ti } } + /// + /// Creates an using the supplied without + /// capturing the current . This overload accepts a + /// parameterless for callbacks that do not require state. + /// + /// The time provider used to create the timer. + /// The parameterless delegate invoked when the timer fires. + /// + /// The amount of time to wait before the first invocation, or + /// to create the timer disarmed. + /// The interval between invocations, or + /// to disable periodic signaling. + /// An created by . + internal static ITimer UnsafeCreateTimer( + TimeProvider timeProvider, + Action callback, + T state, + TimeSpan dueTime, + TimeSpan period) + { + if (ExecutionContext.IsFlowSuppressed()) + { + return timeProvider.CreateTimer(s => callback((T)s!), state, dueTime, period); + } + + using (ExecutionContext.SuppressFlow()) + { + return timeProvider.CreateTimer(s => callback((T)s!), state, dueTime, period); + } + } + + +#nullable restore #region COM+ exceptions internal static ArgumentException Argument(string error) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/BlockingPeriodErrorState.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/BlockingPeriodErrorState.cs new file mode 100644 index 0000000000..47d0156a7e --- /dev/null +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/BlockingPeriodErrorState.cs @@ -0,0 +1,250 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading; +using Microsoft.Data.Common; +using Microsoft.Data.SqlClient.Internal; + +#nullable enable + +namespace Microsoft.Data.SqlClient.ConnectionPool +{ + /// + /// Encapsulates a connection pool's blocking-period error state: cached exception, exponential + /// backoff timer, and synchronization. Kept as a separate class so the pool's + /// connection-acquisition path remains focused on capacity/queue concerns and stays + /// decoupled from the (independent) rate limiting policy. + /// + internal sealed class BlockingPeriodErrorState : IDisposable + { + // Backoff interval used the first time the pool enters the blocking period, and the + // value the backoff resets to on a successful create or Clear(). + private static readonly TimeSpan InitialWait = TimeSpan.FromSeconds(5); + + // Upper bound the exponential backoff is capped at; further failures never wait longer. + private static readonly TimeSpan MaxWait = TimeSpan.FromSeconds(60); + + // Identifier of the owning pool, included in trace events for diagnostics. + private readonly int _ownerPoolId; + + // Optional callback invoked (under _lock) when the state enters the blocking period; + // Must be cheap and non-reentrant. Must not throw. If null, no action is taken. + private readonly Action? _onEnter; + + // Optional callback invoked (under _lock) when the state leaves the blocking period + // via the exit timer or Clear(). Must be cheap and non-reentrant. Must not throw. + // If null, no action is taken. + private readonly Action? _onExit; + + // Time source used to create the exit timer; overridable so tests can drive scheduling + // deterministically. Defaults to TimeProvider.System. + private readonly TimeProvider _timeProvider; + + // Guards the mutable error state (_cachedException, _exitTimer, _nextWait, _disposed). + private readonly object _lock = new(); + + // Non-null while the pool is in the blocking period. Doubles as the "has error" + // flag, so callers don't need a separate bool. Volatile so other threads observe + // entry/exit transitions without acquiring _lock. + private volatile Exception? _cachedException; + + // True from the moment Enter() activates the blocking period until Clear()/Dispose() + // fully resets it. The exit timer clears _cachedException but leaves this set so a + // later Clear() still resets the backoff. Volatile so Clear() can take a + // lock-free path when there is nothing to do. This allows Clear() to be called on hot paths. + private volatile bool _inElevatedState; + + // The armed exit timer that ends the current blocking period; null when no period is + // active. Replaced (and the old one disposed) each time Enter() is called. + private ITimer? _exitTimer; + + // The backoff interval to use for the next Enter(); doubles per failure up to MaxWait + // and resets to InitialWait on a successful create or Clear(). + private TimeSpan _nextWait = InitialWait; + + // True once Dispose() has run, so repeated disposal and post-teardown work are no-ops. + private bool _disposed; + + /// + /// Creates a new instance. + /// + /// Identifier of the owning pool, used in trace events. + /// Optional callback invoked (while holding the internal lock) after + /// the state transitions into the blocking period. Used by the legacy wait-handle pool to + /// signal its error wait handle. Because it runs under a lock, it must be cheap and + /// non-reentrant. + /// Optional callback invoked (while holding the internal lock) after the + /// state transitions out of the blocking period via the exit timer or . + /// Because it runs under a lock, it must be cheap and non-reentrant. + /// The time provider used to create the exit timer. Defaults to + /// . Inject a test double (e.g. + /// Microsoft.Extensions.Time.Testing.FakeTimeProvider) in unit tests to + /// control timer scheduling deterministically. + internal BlockingPeriodErrorState(int ownerPoolId, Action? onEnter = null, Action? onExit = null, TimeProvider? timeProvider = null) + { + _ownerPoolId = ownerPoolId; + _onEnter = onEnter; + _onExit = onExit; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + /// + /// True while the pool is in the blocking period. Subsequent acquisition attempts + /// should fast-fail with the cached exception. + /// + internal bool HasError => _cachedException is not null; + + /// + /// Throws the cached error if the pool is currently in the blocking period. + /// + internal void ThrowIfActive() + { + Exception? cached = _cachedException; + if (cached is null) + { + return; + } + + // Clone SqlExceptions so stack traces are not shared across callers; other + // exception types are rethrown as-is. + throw cached is SqlException sqlEx ? sqlEx.InternalClone() : cached; + } + + /// + /// Enters the blocking period, caching the supplied exception and scheduling a timer + /// to exit the period after the current backoff interval. Subsequent failures double + /// the backoff up to . + /// + internal void Enter(Exception ex) + { + TimeSpan wait; + ITimer? oldTimer; + + // If we call this, we're already in an exception path. Prefer correctness over performance. + lock (_lock) + { + _inElevatedState = true; + _cachedException = ex; + wait = _nextWait; + + ITimer newTimer = ADP.UnsafeCreateTimer( + _timeProvider, + ExitCallback, + this, + wait, + wait); + oldTimer = _exitTimer; + _exitTimer = newTimer; + + // Bump the backoff for the next failure, capped at MaxWait. FR-008. + TimeSpan doubled = _nextWait + _nextWait; + _nextWait = doubled > MaxWait ? MaxWait : doubled; + + // Signal the enter event while still holding the lock so the external signal order + // (onEnter/onExit) can never diverge from the internal state transitions under + // concurrent Enter/Clear/exit-timer activity. The callbacks are expected to be + // cheap, non-reentrant operations. + _onEnter?.Invoke(); + } + + oldTimer?.Dispose(); + + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Entering blocking period for {1}ms.", + _ownerPoolId, + (int)wait.TotalMilliseconds); + } + + /// + /// Clears the cached error state, disposes the exit timer, and resets the backoff to + /// its initial value. + /// + internal void Clear() + { + // Fast path: the create flow calls Clear() after every successful create, so avoid + // taking the lock in the common (no-error) case where there is nothing to reset. + if (!_inElevatedState) + { + return; + } + + ITimer? oldTimer; + + lock (_lock) + { + _cachedException = null; + _nextWait = InitialWait; + _inElevatedState = false; + oldTimer = _exitTimer; + _exitTimer = null; + + // Signal and trace under the lock so the exit signal is ordered consistently + // with the state transition relative to concurrent Enter/exit-timer callbacks. + _onExit?.Invoke(); + } + + oldTimer?.Dispose(); + + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Error state cleared.", _ownerPoolId); + } + + /// + /// Timer callback that exits the blocking period by clearing the cached exception, + /// allowing the next caller to attempt a fresh connection creation. The current + /// backoff is left intact so that, if the next attempt fails, the backoff continues + /// to grow rather than resetting. The backoff is reset only on . + /// + private static void ExitCallback(BlockingPeriodErrorState state) + { + ITimer? oldTimer; + + lock (state._lock) + { + state._cachedException = null; + oldTimer = state._exitTimer; + state._exitTimer = null; + + // Signal under the lock so the exit signal is ordered consistently + // with the state transition relative to concurrent Enter/Clear callbacks. + try { + state._onExit?.Invoke(); + } catch { + // Ignore exceptions from the exit event. + } + } + + oldTimer?.Dispose(); + + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Exiting blocking period.", state._ownerPoolId); + } + + /// + /// Disposes the instance, releasing the exit timer if one is active. Clears the + /// error state so that any waiting callers do not observe a stale exception after + /// the owning pool is torn down. + /// + public void Dispose() + { + ITimer? timerToDispose; + lock (_lock) + { + if (_disposed) + { + return; + } + + _cachedException = null; + _inElevatedState = false; + timerToDispose = _exitTimer; + _exitTimer = null; + } + + timerToDispose?.Dispose(); + _disposed = true; + } + } +} diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/DbConnectionPoolGroup.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/DbConnectionPoolGroup.cs index 8f62787e30..085d300903 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/DbConnectionPoolGroup.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/DbConnectionPoolGroup.cs @@ -93,6 +93,20 @@ internal DbConnectionPoolGroupProviderInfo ProviderInfo internal SqlMetaDataFactory MetaDataFactory { get; set; } + /// + /// Determines whether the blocking period is enabled for this pool group based on the + /// configured and the target data source. + /// + internal bool IsBlockingPeriodEnabled() + { + return _connectionOptions.PoolBlockingPeriod switch + { + PoolBlockingPeriod.Auto => !ADP.IsAzureSqlServerEndpoint(_connectionOptions.DataSource), + PoolBlockingPeriod.AlwaysBlock => true, + PoolBlockingPeriod.NeverBlock => false + }; + } + internal int Clear() { // must be multi-thread safe with competing calls by Clear and Prune via background thread diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs index d2c3b57323..b214193fd9 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs @@ -29,7 +29,7 @@ namespace Microsoft.Data.SqlClient.ConnectionPool /// Transaction-Aware Pooling: Tracks connections enlisted in using TransactedConnectionPool and TransactedConnectionList, ensuring proper context reuse. /// Concurrency and Synchronization: Uses wait handles and semaphores via PoolWaitHandles to coordinate safe multi-threaded access. /// Connection Lifecycle Management: Manages creation (CreateObject), deactivation (DeactivateObject), destruction (DestroyObject), and reclamation (ReclaimEmancipatedObjects) of internal connections. - /// Error Handling and Resilience: Implements retry and exponential backoff in TryGetConnection and handles transient errors using _errorWait. + /// Error Handling and Resilience: Implements retry and exponential backoff in TryGetConnection and delegates blocking-period bookkeeping (cached exception, exit timer) to . /// Minimum Pool Size Enforcement: Maintains the MinPoolSize by spawning background tasks to create new connections when needed. /// Load Balancing Support: Honors LoadBalanceTimeout to clean up idle connections and distribute load evenly. /// Telemetry and Tracing: Uses SqlClientEventSource for extensive diagnostic tracing of connection lifecycle events. @@ -165,8 +165,6 @@ public void Dispose() private const int WAIT_ABANDONED = 0x80; - private const int ERROR_WAIT_DEFAULT = 5 * 1000; // 5 seconds - // we do want a testable, repeatable set of generated random numbers private static readonly Random s_random = new Random(5101977); // Value obtained from Dave Driver @@ -194,11 +192,8 @@ public void Dispose() private int _waitCount; private readonly PoolWaitHandles _waitHandles; - private Exception _resError; - private volatile bool _errorOccurred; - - private int _errorWait; - private Timer _errorTimer; + private readonly TimeProvider _timeProvider; + private readonly BlockingPeriodErrorState _errorState; private Timer _cleanupTimer; @@ -212,7 +207,8 @@ internal WaitHandleDbConnectionPool( SqlConnectionFactory connectionFactory, DbConnectionPoolGroup connectionPoolGroup, DbConnectionPoolIdentity identity, - DbConnectionPoolProviderInfo connectionPoolProviderInfo) + DbConnectionPoolProviderInfo connectionPoolProviderInfo, + TimeProvider timeProvider = null) { Debug.Assert(connectionPoolGroup != null, "null connectionPoolGroup"); @@ -249,11 +245,19 @@ internal WaitHandleDbConnectionPool( _connectionPoolGroupOptions = connectionPoolGroup.PoolGroupOptions; _connectionPoolProviderInfo = connectionPoolProviderInfo; _identity = identity; + _timeProvider = timeProvider ?? TimeProvider.System; _waitHandles = new PoolWaitHandles(); - _errorWait = ERROR_WAIT_DEFAULT; - _errorTimer = null; // No error yet. + // Hook the wait-handle event so any thread blocked in WaitAny over the pool's + // handles wakes up immediately when the blocking period is entered/exited. + // _timeProvider is the system clock in production; tests inject a fake clock to + // drive the exit timer deterministically. + _errorState = new BlockingPeriodErrorState( + Id, + onEnter: () => _waitHandles.ErrorEvent.Set(), + onExit: () => _waitHandles.ErrorEvent.Reset(), + timeProvider: _timeProvider); _objectList = new List(MaxPoolSize); @@ -283,7 +287,7 @@ private int CreationTimeout public SqlConnectionFactory ConnectionFactory => _connectionFactory; - public bool ErrorOccurred => _errorOccurred; + public bool ErrorOccurred => _errorState.HasError; private bool HasTransactionAffinity => PoolGroupOptions.HasTransactionAffinity; @@ -515,39 +519,6 @@ private Timer CreateCleanupTimer() => _cleanupWait, _cleanupWait); - private bool IsBlockingPeriodEnabled() - { - var poolGroupConnectionOptions = _connectionPoolGroup.ConnectionOptions; - if (poolGroupConnectionOptions == null) - { - return true; - } - - var policy = poolGroupConnectionOptions.PoolBlockingPeriod; - - switch (policy) - { - case PoolBlockingPeriod.Auto: - { - return !ADP.IsAzureSqlServerEndpoint(poolGroupConnectionOptions.DataSource); - } - case PoolBlockingPeriod.AlwaysBlock: - { - return true; //Enabled - } - case PoolBlockingPeriod.NeverBlock: - { - return false; //Disabled - } - default: - { - //we should never get into this path. - Debug.Fail("Unknown PoolBlockingPeriod. Please specify explicit results in above switch case statement."); - return true; - } - } - } - private DbConnectionInternal CreateObject(DbConnection owningObject, DbConnectionInternal oldConnection, TimeoutTimer timeout) { DbConnectionInternal newObj = null; @@ -573,14 +544,14 @@ private DbConnectionInternal CreateObject(DbConnection owningObject, DbConnectio SqlClientEventSource.Log.TryPoolerTraceEvent(" {0}, Connection {1}, Added to pool.", Id, newObj?.ObjectID); - // Reset the error wait: - _errorWait = ERROR_WAIT_DEFAULT; + // A successful creation clears any prior error state and resets backoff. + _errorState.Clear(); } catch (Exception e) when (ADP.IsCatchableExceptionType(e)) { ADP.TraceExceptionWithoutRethrow(e); - if (!IsBlockingPeriodEnabled()) + if (!_connectionPoolGroup.IsBlockingPeriodEnabled()) { throw; } @@ -593,33 +564,10 @@ private DbConnectionInternal CreateObject(DbConnection owningObject, DbConnectio newObj = null; // set to null, so we do not return bad new object - // Failed to create instance - _resError = e; - - // Make sure the timer starts even if ThreadAbort occurs after setting the ErrorEvent. - Timer t = new Timer(new TimerCallback(this.ErrorCallback), null, Timeout.Infinite, Timeout.Infinite); - - bool timerIsNotDisposed; - - _waitHandles.ErrorEvent.Set(); - _errorOccurred = true; - - // Enable the timer. - // Note that the timer is created to allow periodic invocation. If ThreadAbort occurs in the middle of ErrorCallback, - // the timer will restart. Otherwise, the timer callback (ErrorCallback) destroys the timer after resetting the error to avoid second callback. - _errorTimer = t; - timerIsNotDisposed = t.Change(_errorWait, _errorWait); - - Debug.Assert(timerIsNotDisposed, "ErrorCallback timer has been disposed"); + // Enter the blocking period: caches the exception, schedules the exit timer, + // and signals the wait-handle error event via the onEnter callback. + _errorState.Enter(e); - if (30000 < _errorWait) - { - _errorWait = 60000; - } - else - { - _errorWait *= 2; - } throw; } return newObj; @@ -792,28 +740,6 @@ private void DestroyObject(DbConnectionInternal obj) } } - private void ErrorCallback(object state) - { - SqlClientEventSource.Log.TryPoolerTraceEvent(" {0}, Resetting Error handling.", Id); - _errorOccurred = false; - _waitHandles.ErrorEvent.Reset(); - - // the error state is cleaned, destroy the timer to avoid periodic invocation - Timer t = _errorTimer; - _errorTimer = null; - if (t != null) - { - t.Dispose(); // Cancel timer request. - } - } - - - private Exception TryCloneCachedException() - // Cached exception can be of any type, so is not always cloneable. - // This functions clones SqlException - // OleDb and Odbc connections are not passing throw this code - => _resError is SqlException sqlEx ? sqlEx.InternalClone() : _resError; - private void WaitForPendingOpen() { Debug.Assert(!Thread.CurrentThread.IsThreadPoolThread, "This thread may block for a long time. Threadpool threads should not be used."); @@ -1037,7 +963,12 @@ private bool TryGetConnection(DbConnection owningObject, uint waitForMultipleObj // Throw the error that PoolCreateRequest stashed. SqlClientEventSource.Log.TryPoolerTraceEvent(" {0}, Errors are set.", Id); Interlocked.Decrement(ref _waitCount); - throw TryCloneCachedException(); + _errorState.ThrowIfActive(); + // Narrow race: error state cleared between WaitAny observing + // the signal and this check. Re-balance _waitCount and let the + // outer do/while loop retry. + Interlocked.Increment(ref _waitCount); + break; case CREATION_HANDLE: SqlClientEventSource.Log.TryPoolerTraceEvent(" {0}, Creating new connection.", Id); @@ -1616,25 +1547,20 @@ private DbConnectionInternal UserCreateRequest(DbConnection owningObject, Timeou // instead obtained creation mutex DbConnectionInternal obj = null; - if (ErrorOccurred) - { - throw TryCloneCachedException(); - } - else + _errorState.ThrowIfActive(); + + if ((oldConnection != null) || (Count < MaxPoolSize) || (0 == MaxPoolSize)) { - if ((oldConnection != null) || (Count < MaxPoolSize) || (0 == MaxPoolSize)) - { - // If we have an odd number of total objects, reclaim any dead objects. - // If we did not find any objects to reclaim, create a new one. + // If we have an odd number of total objects, reclaim any dead objects. + // If we did not find any objects to reclaim, create a new one. - // TODO: Consider implement a control knob here; why do we only check for dead objects ever other time? why not every 10th time or every time? - if ((oldConnection != null) || (Count & 0x1) == 0x1 || !ReclaimEmancipatedObjects()) - { - obj = CreateObject(owningObject, oldConnection, timeout); - } + // TODO: Consider implement a control knob here; why do we only check for dead objects ever other time? why not every 10th time or every time? + if ((oldConnection != null) || (Count & 0x1) == 0x1 || !ReclaimEmancipatedObjects()) + { + obj = CreateObject(owningObject, oldConnection, timeout); } - return obj; } + return obj; } } } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/BlockingPeriodErrorStateTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/BlockingPeriodErrorStateTest.cs new file mode 100644 index 0000000000..157249f5f9 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/BlockingPeriodErrorStateTest.cs @@ -0,0 +1,688 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading; +using Microsoft.Extensions.Time.Testing; +using Microsoft.Data.SqlClient.ConnectionPool; +using Xunit; + +#nullable enable + +namespace Microsoft.Data.SqlClient.UnitTests.ConnectionPool +{ + /// + /// Comprehensive unit tests for covering: + /// - Initial state and error caching + /// - and exception handling + /// - and state reset + /// - Exponential backoff progression (verified with ) + /// - Timer-driven exit behavior (verified with ) + /// - implementation and timer cleanup + /// - Callback invocation and re-entrancy safety + /// + public class BlockingPeriodErrorStateTest + { + #region HasError / initial state + + /// + /// Verifies that a newly constructed has + /// set to false. + /// + [Fact] + public void HasError_InitialState_IsFalse() + { + // Arrange + using var state = new BlockingPeriodErrorState(ownerPoolId: 1); + + // Act & Assert + Assert.False(state.HasError); + } + + /// + /// Verifies that does not throw + /// when called on a newly constructed instance with no cached error. + /// + [Fact] + public void ThrowIfActive_InitialState_DoesNotThrow() + { + // Arrange + using var state = new BlockingPeriodErrorState(ownerPoolId: 1); + + // Act & Assert + state.ThrowIfActive(); // Should complete without throwing + } + + #endregion + + #region Enter + + /// + /// Verifies that calling sets + /// to true. + /// + [Fact] + public void Enter_SetsHasErrorToTrue() + { + // Arrange + using var state = new BlockingPeriodErrorState(ownerPoolId: 1); + + // Act + state.Enter(new InvalidOperationException("test")); + + // Assert + Assert.True(state.HasError); + } + + /// + /// Verifies that throws + /// the exact exception type that was cached by . + /// + [Fact] + public void Enter_ThrowIfActive_ThrowsCachedExceptionType() + { + // Arrange + using var state = new BlockingPeriodErrorState(ownerPoolId: 1); + var exception = new InvalidOperationException("boom"); + + // Act + state.Enter(exception); + + // Assert + var ex = Assert.Throws(() => state.ThrowIfActive()); + Assert.Equal("boom", ex.Message); + } + + /// + /// Verifies that when a is cached, + /// throws a cloned instance rather than the original, to avoid sharing stack traces across callers. + /// + [Fact] + public void Enter_WithSqlException_ThrowsClonedInstance() + { + // Arrange + using var state = new BlockingPeriodErrorState(ownerPoolId: 1); + var original = SqlExceptionHelper.CreateSqlException("connection failed"); + + // Act + state.Enter(original); + var thrown = Assert.Throws(() => state.ThrowIfActive()); + + // Assert + Assert.NotSame(original, thrown); + Assert.Equal(original.Message, thrown.Message); + } + + /// + /// Verifies that invokes the optional + /// onEnter callback after entering the blocking period. + /// + [Fact] + public void Enter_InvokesOnEnterCallback() + { + // Arrange + int callCount = 0; + using var state = new BlockingPeriodErrorState(ownerPoolId: 1, onEnter: () => callCount++); + + // Act + state.Enter(new Exception()); + + // Assert + Assert.Equal(1, callCount); + } + + /// + /// Verifies that calling a second time + /// replaces the cached exception, invokes the callback again, and the new exception is thrown. + /// + [Fact] + public void Enter_CalledTwice_ReplacesExceptionAndInvokesOnEnterAgain() + { + // Arrange + int enterCount = 0; + using var state = new BlockingPeriodErrorState(ownerPoolId: 1, onEnter: () => enterCount++); + + // Act + state.Enter(new InvalidOperationException("first")); + state.Enter(new ArgumentException("second")); + + // Assert + Assert.Equal(2, enterCount); + var ex = Assert.Throws(() => state.ThrowIfActive()); + Assert.Equal("second", ex.Message); + } + + /// + /// Verifies that does not invoke + /// the onExit callback (only the onEnter callback or the timer should trigger onExit). + /// + [Fact] + public void Enter_DoesNotInvokeOnExitCallback() + { + // Arrange + int exitCount = 0; + using var state = new BlockingPeriodErrorState(ownerPoolId: 1, onExit: () => exitCount++); + + // Act + state.Enter(new Exception()); + + // Assert + Assert.Equal(0, exitCount); + } + + #endregion + + #region Clear + + /// + /// Verifies that resets + /// to false. + /// + [Fact] + public void Clear_AfterEnter_ResetsHasError() + { + // Arrange + using var state = new BlockingPeriodErrorState(ownerPoolId: 1); + state.Enter(new Exception()); + + // Act + state.Clear(); + + // Assert + Assert.False(state.HasError); + } + + /// + /// Verifies that after , + /// does not throw because the cached error has been cleared. + /// + [Fact] + public void Clear_AfterEnter_ThrowIfActiveDoesNotThrow() + { + // Arrange + using var state = new BlockingPeriodErrorState(ownerPoolId: 1); + state.Enter(new Exception()); + + // Act + state.Clear(); + + // Assert + state.ThrowIfActive(); // Must not throw + } + + /// + /// Verifies that invokes the optional + /// onExit callback after clearing the error state. + /// + [Fact] + public void Clear_AfterEnter_InvokesOnExitCallback() + { + // Arrange + int exitCount = 0; + using var state = new BlockingPeriodErrorState(ownerPoolId: 1, onExit: () => exitCount++); + state.Enter(new Exception()); + + // Act + state.Clear(); + + // Assert + Assert.Equal(1, exitCount); + } + + /// + /// Verifies that on an initial (no-error) state + /// does not invoke the onExit callback because there is nothing to clear. + /// + [Fact] + public void Clear_OnInitialState_DoesNotInvokeOnExitCallback() + { + // Arrange + int exitCount = 0; + using var state = new BlockingPeriodErrorState(ownerPoolId: 1, onExit: () => exitCount++); + + // Act + state.Clear(); + + // Assert + Assert.Equal(0, exitCount); + } + + /// + /// Verifies that is idempotent: + /// calling it a second time does not invoke the onExit callback again. + /// + [Fact] + public void Clear_CalledTwice_OnExitInvokedOnce() + { + // Arrange + int exitCount = 0; + using var state = new BlockingPeriodErrorState(ownerPoolId: 1, onExit: () => exitCount++); + state.Enter(new Exception()); + + // Act + state.Clear(); + state.Clear(); + + // Assert + Assert.Equal(1, exitCount); + } + + /// + /// Verifies that resets the backoff + /// timeout to its initial value, so the next + /// uses the initial wait duration instead of the accumulated backoff. + /// + [Fact] + public void Clear_ResetsBackoffSoNextEnterUsesInitialWait() + { + // Arrange + using var state = new BlockingPeriodErrorState(ownerPoolId: 1); + state.Enter(new Exception("first")); + + // Act + state.Clear(); + state.Enter(new Exception("second")); + + // Assert + Assert.True(state.HasError); + } + + #endregion + + #region Backoff progression + + /// + /// Verifies that the initial enter schedules the timer with the 5-second initial wait. + /// The error state should persist until the timer fires, after which it clears automatically. + /// + [Fact] + public void Enter_FirstEntry_SchedulesInitialWaitTimer() + { + // Arrange + var fakeTime = new FakeTimeProvider(); + using var state = new BlockingPeriodErrorState(ownerPoolId: 1, timeProvider: fakeTime); + state.Enter(new Exception()); + + // Act: advance just under the initial 5s wait + fakeTime.Advance(TimeSpan.FromSeconds(4)); + + // Assert: timer has not fired yet + Assert.True(state.HasError); + + // Act: advance past the due time + fakeTime.Advance(TimeSpan.FromSeconds(1)); + + // Assert: timer has fired, error cleared + Assert.False(state.HasError); + } + + /// + /// Verifies that successive timer-driven exits double the backoff each time: + /// 5 s → 10 s → 20 s → 40 s → 60 s (capped at MaxWait). + /// Each Enter schedules the timer for the current accumulated wait and the error + /// persists until exactly that duration elapses. + /// + [Fact] + public void Enter_BackoffDoublesOnSuccessiveTimerFiredEntries() + { + // Arrange + var fakeTime = new FakeTimeProvider(); + using var state = new BlockingPeriodErrorState(ownerPoolId: 1, timeProvider: fakeTime); + + // (expectedWaitSeconds, _) — the wait used by Enter for that iteration + int[] expectedWaits = [5, 10, 20, 40, 60, 60]; + + // Act & Assert + foreach (int wait in expectedWaits) + { + state.Enter(new Exception($"attempt at wait={wait}s")); + + // One second before due time: error still active + fakeTime.Advance(TimeSpan.FromSeconds(wait - 1)); + Assert.True(state.HasError, $"HasError should be true after {wait - 1}s (scheduled wait={wait}s)"); + + // Final second: timer fires, error clears + fakeTime.Advance(TimeSpan.FromSeconds(1)); + Assert.False(state.HasError, $"HasError should be false after {wait}s (scheduled wait={wait}s)"); + } + } + + /// + /// Verifies that a timer-driven exit does NOT reset the backoff. The accumulated + /// backoff is preserved so the next failure uses the doubled wait, reflecting + /// continued instability. Only resets + /// the backoff to the initial value. In this way, we only reset the backoff when + /// a connection is successfully established. + /// + [Fact] + public void Enter_WhenTimerFires_DoesNotResetBackoff() + { + // Arrange + var fakeTime = new FakeTimeProvider(); + using var state = new BlockingPeriodErrorState(ownerPoolId: 1, timeProvider: fakeTime); + + // First enter: uses 5s wait; _nextWait advances to 10s + state.Enter(new Exception("first")); + fakeTime.Advance(TimeSpan.FromSeconds(5)); // timer fires + Assert.False(state.HasError); + + // Act: enter again — should use 10s, not the initial 5s + state.Enter(new Exception("second")); + + // Assert: not cleared after 9s + fakeTime.Advance(TimeSpan.FromSeconds(9)); + Assert.True(state.HasError); + + // Assert: cleared after the full 10s + fakeTime.Advance(TimeSpan.FromSeconds(1)); + Assert.False(state.HasError); + } + + /// + /// Verifies that resets the backoff + /// to the initial 5-second wait even after the timer has doubled it, so the next + /// enter cycle starts fresh. + /// + [Fact] + public void Clear_AfterTimerFiredEntry_ResetsBackoffToInitialWait() + { + // Arrange + var fakeTime = new FakeTimeProvider(); + using var state = new BlockingPeriodErrorState(ownerPoolId: 1, timeProvider: fakeTime); + + // First enter/timer-exit: _nextWait advances from 5s to 10s + state.Enter(new Exception("first")); + fakeTime.Advance(TimeSpan.FromSeconds(5)); + Assert.False(state.HasError); + + // Second enter to accumulate more backoff; then Clear resets it + state.Enter(new Exception("second")); // _nextWait advances to 20s + state.Clear(); // _nextWait resets to 5s + + // Act: enter again — should use the initial 5s wait + state.Enter(new Exception("third")); + + // Assert: not cleared after 4s + fakeTime.Advance(TimeSpan.FromSeconds(4)); + Assert.True(state.HasError); + + // Assert: cleared after the initial 5s + fakeTime.Advance(TimeSpan.FromSeconds(1)); + Assert.False(state.HasError); + } + + #endregion + + #region Timer behavior + + /// + /// Verifies that the timer-driven exit invokes the onExit callback, the same + /// callback path used by . + /// + [Fact] + public void Enter_WhenTimerFires_InvokesOnExitCallback() + { + // Arrange + var fakeTime = new FakeTimeProvider(); + int exitCount = 0; + using var state = new BlockingPeriodErrorState( + ownerPoolId: 1, + onExit: () => exitCount++, + timeProvider: fakeTime); + state.Enter(new Exception()); + + // Act + fakeTime.Advance(TimeSpan.FromSeconds(5)); + + // Assert + Assert.Equal(1, exitCount); + Assert.False(state.HasError); + } + + /// + /// Verifies that the timer does not fire before its due time, confirming the + /// scheduled wait is respected and not fired early. + /// + [Fact] + public void Enter_TimerDoesNotFireBeforeDueTime() + { + // Arrange + var fakeTime = new FakeTimeProvider(); + using var state = new BlockingPeriodErrorState(ownerPoolId: 1, timeProvider: fakeTime); + state.Enter(new Exception()); + + // Act: advance to 1ms before the 5s due time + fakeTime.Advance(TimeSpan.FromMilliseconds(4999)); + + // Assert + Assert.True(state.HasError); + } + + #endregion + + #region IDisposable + + /// + /// Verifies that on an initial state + /// does not throw and completes successfully. + /// + [Fact] + public void Dispose_OnInitialState_DoesNotThrow() + { + // Arrange & Act + var state = new BlockingPeriodErrorState(ownerPoolId: 1); + + // Assert + state.Dispose(); // Should not throw + } + + /// + /// Verifies that clears the cached + /// error state so is false after disposal. + /// + [Fact] + public void Dispose_AfterEnter_ClearsHasError() + { + // Arrange + var state = new BlockingPeriodErrorState(ownerPoolId: 1); + state.Enter(new Exception()); + + // Act + state.Dispose(); + + // Assert + Assert.False(state.HasError); + } + + /// + /// Verifies that is idempotent: + /// calling it multiple times does not throw and completes successfully. + /// + [Fact] + public void Dispose_CalledMultipleTimes_DoesNotThrow() + { + // Arrange & Act + var state = new BlockingPeriodErrorState(ownerPoolId: 1); + state.Dispose(); + + // Assert + state.Dispose(); // Must be idempotent and not throw + } + + /// + /// Verifies that does not invoke + /// the onExit callback because disposal is a resource-cleanup path, not a logical + /// "exit blocking period" event. + /// + [Fact] + public void Dispose_DoesNotInvokeOnExitCallback() + { + // Arrange + int exitCount = 0; + var state = new BlockingPeriodErrorState(ownerPoolId: 1, onExit: () => exitCount++); + state.Enter(new Exception()); + + // Act + state.Dispose(); + + // Assert + Assert.Equal(0, exitCount); + } + + /// + /// Verifies that properly releases + /// and cancels the internal exit timer, preventing stale callbacks from firing after disposal. + /// Uses to advance time deterministically past the timer's due + /// time without relying on real-time sleeps. + /// + [Fact] + public void Dispose_ReleasesTimer_NoCallbackAfterDispose() + { + // Arrange + var fakeTime = new FakeTimeProvider(); + int exitCount = 0; + var state = new BlockingPeriodErrorState( + ownerPoolId: 1, + onExit: () => Interlocked.Increment(ref exitCount), + timeProvider: fakeTime); + state.Enter(new Exception()); + + // Act: dispose cancels the pending timer + state.Dispose(); + + // Advance well past the 5s due time — the cancelled timer must not fire + fakeTime.Advance(TimeSpan.FromSeconds(60)); + + // Assert + Assert.Equal(0, exitCount); + } + + /// + /// Verifies that works correctly in a standard + /// using statement, with no exceptions thrown during disposal. + /// + [Fact] + public void Dispose_WithUsingStatement_DoesNotThrow() + { + // Arrange & Act + using (var state = new BlockingPeriodErrorState(ownerPoolId: 1)) + { + state.Enter(new Exception()); + Assert.True(state.HasError); + } + + // Assert + // No exception expected when the using block exits + } + + #endregion + + #region Callback behaviour + + /// + /// Verifies that both onEnter and onExit callbacks are optional (nullable) + /// and the instance works correctly when neither is provided. + /// + [Fact] + public void Callbacks_AreNotRequiredAndDefaultToNull() + { + // Arrange & Act + using var state = new BlockingPeriodErrorState(ownerPoolId: 42); + state.Enter(new Exception()); + + // Assert + state.Clear(); // Should work without callbacks + } + + /// + /// Verifies that the onEnter callback is invoked after the internal lock is released. + /// The callback reads and calls + /// — operations that are safe only + /// when the lock is not held. If the implementation were changed to hold the lock during + /// the callback invocation, any re-entrant call from the callback that tries to acquire the + /// same lock (on a non-re-entrant lock) would deadlock. + /// + [Fact] + public void OnEnter_CalledOutsideLock_CanCallBackIntoState() + { + // Arrange + bool hasErrorObservedInCallback = false; + BlockingPeriodErrorState? state = null; + using (state = new BlockingPeriodErrorState( + ownerPoolId: 1, + onEnter: () => + { + // Observe state from inside the callback. + // HasError must already be true at this point. + hasErrorObservedInCallback = state!.HasError; + + // Calling ThrowIfActive from the callback must not deadlock. + Assert.Throws(() => state.ThrowIfActive()); + })) + { + + // Act + state.Enter(new InvalidOperationException("test")); + } + + // Assert + Assert.True(hasErrorObservedInCallback); + } + + /// + /// Verifies that the onExit callback is invoked after the internal lock is released. + /// The callback reads — confirming it + /// observes the cleared state — and calls + /// without deadlocking. If the implementation were changed to hold the lock during the + /// callback, any re-entrant call from the callback would deadlock on a non-re-entrant lock. + /// + [Fact] + public void OnExit_CalledOutsideLock_CanCallBackIntoState() + { + // Arrange + bool hasErrorObservedInCallback = true; // initialized to true; must be false after Clear + BlockingPeriodErrorState? state = null; + using (state = new BlockingPeriodErrorState( + ownerPoolId: 1, + onExit: () => + { + // HasError must already be false when onExit is called. + hasErrorObservedInCallback = state!.HasError; + + // Calling ThrowIfActive from the callback must not deadlock. + state.ThrowIfActive(); // must not throw + })) + { + + // Act + state.Enter(new Exception()); + state.Clear(); + } + + // Assert + Assert.False(hasErrorObservedInCallback); + } + + #endregion + } + + /// + /// Test helper for creating instances. Since has + /// an internal constructor, instances must be created via the factory method. + /// + internal static class SqlExceptionHelper + { + /// + /// Creates a with the specified message using the internal factory method. + /// + /// The error message for the exception. + /// A new with the specified message. + internal static SqlException CreateSqlException(string message) + { + var collection = new SqlErrorCollection(); + collection.Add(new SqlError(0, (byte)0, (byte)0, "TestServer", message, "", 0)); + return SqlException.CreateException(collection, ""); + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs new file mode 100644 index 0000000000..5adb2d8661 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs @@ -0,0 +1,401 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Data.Common; +using System.Threading; +using System.Transactions; +using Microsoft.Data.ProviderBase; +using Microsoft.Data.SqlClient.ConnectionPool; +using Microsoft.Extensions.Time.Testing; +using Xunit; + +#nullable enable + +namespace Microsoft.Data.SqlClient.UnitTests.ConnectionPool; + +/// +/// End-to-end coverage of the blocking-period +/// behavior, exercising the full connection-acquisition path rather than the +/// state machine in isolation. Verifies the wiring +/// between a failed physical connection create and the pool's error state: +/// - A failed create enters the blocking period and surfaces the error (CreateObject → Enter). +/// - Subsequent requests fast-fail with the cached (cloned) exception without re-invoking the +/// connection factory (error wait-handle → ThrowIfActive). +/// - A successful create does not trip the blocking period. +/// - bypasses the blocking period entirely so each +/// request retries the factory. +/// - After the blocking-period exit timer fires, the next request retries the factory and a +/// successful create recovers the pool and resets the backoff (driven deterministically by an +/// injected ). +/// +public class WaitHandleDbConnectionPoolBlockingPeriodTest : IDisposable +{ + private const int DefaultMaxPoolSize = 50; + private const int DefaultMinPoolSize = 0; + private const int DefaultCreationTimeoutInMilliseconds = 15_000; + + private WaitHandleDbConnectionPool? _pool; + + public void Dispose() + { + _pool?.Shutdown(); + _pool?.Clear(); + } + + /// + /// Builds a running backed by the supplied factory. + /// The controls the data source and + /// policy used to resolve whether blocking is enabled. + /// When is supplied, the pool's + /// uses it as its clock so the exit timer can be driven + /// deterministically; otherwise the system clock is used. + /// + private WaitHandleDbConnectionPool CreatePool( + SqlConnectionFactory connectionFactory, + string connectionString = "Data Source=localhost;", + TimeProvider? timeProvider = null) + { + var poolGroupOptions = new DbConnectionPoolGroupOptions( + poolByIdentity: false, + minPoolSize: DefaultMinPoolSize, + maxPoolSize: DefaultMaxPoolSize, + creationTimeout: DefaultCreationTimeoutInMilliseconds, + loadBalanceTimeout: 0, + hasTransactionAffinity: true, + idleTimeout: 0); + + var dbConnectionPoolGroup = new DbConnectionPoolGroup( + new SqlConnectionOptions(connectionString), + new ConnectionPoolKey("TestDataSource", credential: null, accessToken: null, accessTokenCallback: null, sspiContextProvider: null), + poolGroupOptions); + + var pool = new WaitHandleDbConnectionPool( + connectionFactory, + dbConnectionPoolGroup, + DbConnectionPoolIdentity.NoIdentity, + new DbConnectionPoolProviderInfo(), + timeProvider); + + pool.Startup(); + _pool = pool; + return pool; + } + + /// + /// Synchronously requests a connection from the pool, mirroring the sync acquisition path + /// (taskCompletionSource == null) used by callers of SqlConnection.Open. + /// + private static bool TryGetConnectionSync( + WaitHandleDbConnectionPool pool, + DbConnection owner, + out DbConnectionInternal? connection) + { + TimeoutTimer timer = TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)); + return pool.TryGetConnection(owner, taskCompletionSource: null, timer, out connection); + } + + /// + /// Verifies that when the connection factory fails, the pool enters the blocking period: + /// the originating exception is surfaced to the caller and + /// becomes true. Guards the CreateObject → wiring. + /// + [Fact] + public void TryGetConnection_WhenFactoryThrows_EntersBlockingPeriod() + { + // Arrange + SqlException failure = SqlExceptionHelper.CreateSqlException("server unreachable"); + var factory = new ConfigurableSqlConnectionFactory(_ => throw failure); + var pool = CreatePool(factory); + using var owner = new SqlConnection(); + + // Act + SqlException thrown = Assert.Throws( + () => TryGetConnectionSync(pool, owner, out _)); + + // Assert + Assert.Equal(failure.Message, thrown.Message); + Assert.True(pool.ErrorOccurred); + Assert.Equal(1, factory.CreateConnectionCallCount); + } + + /// + /// Verifies that once the pool is in the blocking period, a subsequent request fast-fails + /// with the cached exception without invoking the connection factory again. The first throw + /// rethrows the original instance; the fast-fail throw returns a clone (to avoid sharing stack + /// traces). Guards the error wait-handle → path. + /// + [Fact] + public void TryGetConnection_WhileBlocked_FastFailsWithCachedExceptionWithoutInvokingFactory() + { + // Arrange + SqlException failure = SqlExceptionHelper.CreateSqlException("server unreachable"); + var factory = new ConfigurableSqlConnectionFactory(_ => throw failure); + var pool = CreatePool(factory); + using var owner = new SqlConnection(); + + // Act: first request enters the blocking period and rethrows the original exception. + SqlException first = Assert.Throws( + () => TryGetConnectionSync(pool, owner, out _)); + + // Second request must fast-fail from the cached state without reaching the factory. + SqlException second = Assert.Throws( + () => TryGetConnectionSync(pool, owner, out _)); + + // Assert + Assert.Same(failure, first); // original instance rethrown on entry + Assert.NotSame(failure, second); // cloned on fast-fail + Assert.Equal(failure.Message, second.Message); // but message is preserved + Assert.Equal(1, factory.CreateConnectionCallCount); // factory not re-invoked while blocked + Assert.True(pool.ErrorOccurred); + } + + /// + /// Verifies that with , a failed create does not + /// cache the error: stays false and each + /// request retries the connection factory. Guards the + /// IsBlockingPeriodEnabled() == false bypass in CreateObject. + /// + [Fact] + public void TryGetConnection_WithNeverBlockPolicy_DoesNotCacheErrorAndRetriesFactory() + { + // Arrange + SqlException failure = SqlExceptionHelper.CreateSqlException("server unreachable"); + var factory = new ConfigurableSqlConnectionFactory(_ => throw failure); + var pool = CreatePool(factory, "Data Source=localhost;PoolBlockingPeriod=NeverBlock"); + using var owner = new SqlConnection(); + + // Act: two independent failures, neither of which should be cached. + SqlException first = Assert.Throws(() => TryGetConnectionSync(pool, owner, out _)); + SqlException second = Assert.Throws(() => TryGetConnectionSync(pool, owner, out _)); + + // Assert + Assert.Same(failure, first); // original instance rethrown, never cached/cloned + Assert.Same(failure, second); // original instance rethrown again on retry + Assert.False(pool.ErrorOccurred); + Assert.Equal(2, factory.CreateConnectionCallCount); + } + + /// + /// Verifies that a successful create returns a connection and does not trip the blocking + /// period, confirming the normal path leaves + /// false (and exercises the fast no-op on success). + /// + [Fact] + public void TryGetConnection_WhenFactorySucceeds_DoesNotEnterBlockingPeriod() + { + // Arrange + var factory = new ConfigurableSqlConnectionFactory(_ => new MockDbConnectionInternal()); + var pool = CreatePool(factory); + using var owner = new SqlConnection(); + + // Act + bool completed = TryGetConnectionSync(pool, owner, out DbConnectionInternal? connection); + + // Assert + Assert.True(completed); + Assert.NotNull(connection); + Assert.False(pool.ErrorOccurred); + Assert.Equal(1, factory.CreateConnectionCallCount); + } + + /// + /// Verifies that once the blocking period's exit timer fires, the next request retries the + /// factory and a successful create recovers the pool: + /// returns to false and a connection is produced. Drives the exit timer deterministically with + /// an injected , guarding the + /// timer-exit → retry → Clear path at the pool level. + /// + [Fact] + public void TryGetConnection_AfterBlockingPeriodExpires_RetriesFactoryAndRecovers() + { + // Arrange: first create fails, every later create succeeds. + SqlException failure = SqlExceptionHelper.CreateSqlException("server unreachable"); + var factory = new ConfigurableSqlConnectionFactory( + call => call == 1 ? throw failure : new MockDbConnectionInternal()); + var fakeTime = new FakeTimeProvider(); + var pool = CreatePool(factory, timeProvider: fakeTime); + using var owner = new SqlConnection(); + + // Act: first request enters the blocking period. + Assert.Throws(() => TryGetConnectionSync(pool, owner, out _)); + Assert.True(pool.ErrorOccurred); + Assert.Equal(1, factory.CreateConnectionCallCount); + + // While blocked, a request fast-fails without reaching the factory. + Assert.Throws(() => TryGetConnectionSync(pool, owner, out _)); + Assert.Equal(1, factory.CreateConnectionCallCount); + + // Advance past the initial 5s blocking period so the exit timer fires. + fakeTime.Advance(TimeSpan.FromSeconds(5)); + + // Assert: the pool has recovered and a new request reaches the factory and succeeds. + Assert.False(pool.ErrorOccurred); + bool completed = TryGetConnectionSync(pool, owner, out DbConnectionInternal? connection); + Assert.True(completed); + Assert.NotNull(connection); + Assert.False(pool.ErrorOccurred); + Assert.Equal(2, factory.CreateConnectionCallCount); + } + + /// + /// Verifies that a successful create resets the exponential backoff to the initial 5-second + /// wait. The backoff is first allowed to grow (fail → timer-exit → fail → timer-exit, observing + /// the doubled 10-second wait) so the reset is observable. A successful create then resets the + /// backoff via , and a subsequent failure blocks for + /// only the initial 5 seconds rather than the accumulated 20 seconds, confirming the success + /// reset is wired through the pool. Drives timing deterministically with an injected + /// . + /// + [Fact] + public void TryGetConnection_SuccessfulCreate_ResetsBackoffToInitialWait() + { + // Arrange: creates #1, #2 and #4 fail; #3 succeeds (recovery) to reset the backoff. + SqlException failure = SqlExceptionHelper.CreateSqlException("server unreachable"); + var factory = new ConfigurableSqlConnectionFactory( + call => call == 3 ? new MockDbConnectionInternal() : throw failure); + var fakeTime = new FakeTimeProvider(); + var pool = CreatePool(factory, timeProvider: fakeTime); + using var owner = new SqlConnection(); + + // Act: first failure enters the blocking period with the initial 5s wait. + Assert.Throws(() => TryGetConnectionSync(pool, owner, out _)); + Assert.Equal(1, factory.CreateConnectionCallCount); + fakeTime.Advance(TimeSpan.FromSeconds(5)); // timer fires -> backoff doubles to 10s + Assert.False(pool.ErrorOccurred); + + // Second failure enters with the doubled 10s wait; confirm it lasts the full 10s so the + // backoff has demonstrably grown before we reset it. + Assert.Throws(() => TryGetConnectionSync(pool, owner, out _)); // create #2 fails + Assert.Equal(2, factory.CreateConnectionCallCount); + fakeTime.Advance(TimeSpan.FromSeconds(9)); + Assert.True(pool.ErrorOccurred); // still blocked at 9s -> wait is 10s, not 5s + fakeTime.Advance(TimeSpan.FromSeconds(1)); // timer fires at 10s -> backoff would double to 20s + Assert.False(pool.ErrorOccurred); + + // A successful create resets the backoff to the initial 5s. + Assert.True(TryGetConnectionSync(pool, owner, out _)); // create #3 succeeds -> Clear() + Assert.Equal(3, factory.CreateConnectionCallCount); + + // A new failure enters the blocking period again. + Assert.Throws(() => TryGetConnectionSync(pool, owner, out _)); // create #4 fails + Assert.Equal(4, factory.CreateConnectionCallCount); + Assert.True(pool.ErrorOccurred); + + // Assert: the wait was reset to the initial 5s (not the accumulated 20s). + fakeTime.Advance(TimeSpan.FromSeconds(4)); + Assert.True(pool.ErrorOccurred); // still blocked just before the 5s mark + fakeTime.Advance(TimeSpan.FromSeconds(1)); + Assert.False(pool.ErrorOccurred); // cleared exactly at 5s, proving the backoff reset + } + + /// + /// Verifies that a timer-driven exit alone does NOT reset the backoff: when a failure recurs + /// after the exit timer fires (with no intervening successful create), the next blocking period + /// uses the doubled wait (10s, not the initial 5s). Confirms that only + /// — invoked on a successful create — resets the + /// backoff at the pool level. Drives timing deterministically with an injected + /// . + /// + [Fact] + public void TryGetConnection_FailingAgainAfterExitTimer_StillDoublesBackoff() + { + // Arrange: every create fails, so the backoff is never reset by a success. + SqlException failure = SqlExceptionHelper.CreateSqlException("server unreachable"); + var factory = new ConfigurableSqlConnectionFactory(_ => throw failure); + var fakeTime = new FakeTimeProvider(); + var pool = CreatePool(factory, timeProvider: fakeTime); + using var owner = new SqlConnection(); + + // Act: first failure enters the blocking period (initial 5s wait). + Assert.Throws(() => TryGetConnectionSync(pool, owner, out _)); + Assert.Equal(1, factory.CreateConnectionCallCount); + + // Advance past the 5s wait so the exit timer fires (no success -> backoff not reset). + fakeTime.Advance(TimeSpan.FromSeconds(5)); + Assert.False(pool.ErrorOccurred); + + // A new failure re-enters the blocking period using the doubled 10s wait. The factory is + // invoked again, confirming the request reached creation rather than fast-failing. + Assert.Throws(() => TryGetConnectionSync(pool, owner, out _)); + Assert.Equal(2, factory.CreateConnectionCallCount); + Assert.True(pool.ErrorOccurred); + + // Assert: still blocked at the original 5s mark (proves the wait is not 5s)... + fakeTime.Advance(TimeSpan.FromSeconds(5)); + Assert.True(pool.ErrorOccurred); + + // ...and not cleared until the full doubled 10s elapses. + fakeTime.Advance(TimeSpan.FromSeconds(4)); + Assert.True(pool.ErrorOccurred); + fakeTime.Advance(TimeSpan.FromSeconds(1)); + Assert.False(pool.ErrorOccurred); // cleared exactly at 10s, proving the backoff doubled + } + + /// + /// test double whose CreateConnection behavior is + /// supplied per-call (to throw or return a mock connection) and which records how many times + /// the factory was invoked so tests can assert fast-fail (no re-invocation) versus retry. + /// + internal sealed class ConfigurableSqlConnectionFactory : SqlConnectionFactory + { + private readonly Func _createBehavior; + private int _callCount; + + /// + /// The number of times CreateConnection has been invoked. The argument passed to + /// the behavior delegate is the 1-based invocation index. + /// + internal int CreateConnectionCallCount => Volatile.Read(ref _callCount); + + internal ConfigurableSqlConnectionFactory(Func createBehavior) + => _createBehavior = createBehavior; + + protected override DbConnectionInternal CreateConnection( + SqlConnectionOptions options, + ConnectionPoolKey poolKey, + DbConnectionPoolGroupProviderInfo poolGroupProviderInfo, + IDbConnectionPool pool, + DbConnection owningConnection, + TimeoutTimer timeout) + { + int call = Interlocked.Increment(ref _callCount); + return _createBehavior(call); + } + } + + /// + /// Minimal stub used as a successful create result. + /// Duplicated locally so this test file remains self-contained, mirroring the helpers in the + /// sibling WaitHandleDbConnectionPool test files. + /// + internal sealed class MockDbConnectionInternal : DbConnectionInternal + { + public override string ServerVersion => "Mock"; + + public override DbTransaction BeginTransaction(System.Data.IsolationLevel il) + => throw new NotImplementedException(); + + public override void EnlistTransaction(Transaction? transaction) + { + if (transaction != null) + { + EnlistedTransaction = transaction; + } + } + + protected override void Activate(Transaction? transaction) + { + EnlistedTransaction = transaction; + } + + protected override void Deactivate() + { + } + + internal override void ResetConnection() + { + } + } +}