Details
-
Bug
-
Resolution: Unresolved
-
Major
-
3.3.6
-
None
-
None
-
0
-
SDK08
Description
After upgrading from v2.7 to v3.3.6 seems we are facing a connection leak on our pre-prod environment
Was configured (min: 30 connections, max: 60 connections)
Min repro:
ICouchbaseCollection _couchbaseCollection; var circuitBreakerConfiguration = new CircuitBreakerConfiguration(); |
circuitBreakerConfiguration.Enabled = false; |
uint GetPerOperation = 300;
|
var clusterOptions = new ClusterOptions |
{
|
UserName = "developer", |
Password = "iwannarock", |
NumKvConnections = 1,
|
MaxKvConnections = 3,
|
KvSendQueueCapacity = 150,
|
CircuitBreakerConfiguration = circuitBreakerConfiguration
|
|
};
|
var cluster = Couchbase.Cluster.ConnectAsync("couchbase://localhost", clusterOptions.WithRetryStrategy(new TunningBestEffortRetryStrategy(false, 1, 1, 100))) |
.GetAwaiter()
|
.GetResult(); // get a bucket reference |
var bucket = cluster
|
.BucketAsync("general") |
.GetAwaiter()
|
.GetResult(); bucket.WaitUntilReadyAsync(TimeSpan.MaxValue);
|
_couchbaseCollection = bucket.DefaultCollection();
|
_couchbaseCollection.UpsertAsync("my-document-key", new { Name = "Kaio" }); |
while (true) |
{
|
Console.WriteLine($"{DateTime.Now} - Iterating..."); |
var tasks = new Task[GetPerOperation]; |
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); |
cancellationTokenSource.CancelAfter(1500);
|
|
var token = cancellationTokenSource.Token;
|
token.Register(() => Console.WriteLine("timeout")); |
for (var i = 0; i < GetPerOperation; i++) |
tasks[i] = Task.Run(async () =>
|
{
|
try |
{ await _couchbaseCollection.GetAsync("my-document-key", (options) => options.CancellationToken(token)); |
}
|
catch (Exception e) |
{ } }); Task.WaitAll(tasks);
|
Console.WriteLine($"{DateTime.Now} - End..."); |
Task.Delay(TimeSpan.FromSeconds(5));
|
cancellationTokenSource.Dispose();
|
public class TunningBestEffortRetryStrategy : IRetryStrategy |
{
|
private readonly IBackoffCalculator _backoffCalculator; |
private readonly bool _enabled; |
|
public TunningBestEffortRetryStrategy(bool enable, int maxRetries, int delayMillis, int maxDelayMillis) : |
this(ExponentialBackoff.Create(maxRetries, delayMillis, maxDelayMillis)) => |
_enabled = enable;
|
|
public TunningBestEffortRetryStrategy(IBackoffCalculator calculator) => |
_backoffCalculator = calculator;
|
|
public RetryAction RetryAfter(IRequest request, RetryReason reason) |
{
|
if (!_enabled) |
return RetryAction.Duration(null); |
|
if (reason == RetryReason.NoRetry) |
return RetryAction.Duration(null); |
|
if (request.Idempotent || reason.AllowsNonIdempotentRetries()) |
{
|
var backoffDuration = _backoffCalculator.CalculateBackoff(request);
|
return RetryAction.Duration(backoffDuration); |
}
|
|
return RetryAction.Duration(null); |
}
|
}
|
After take a look in the code we have some ideas that we would like to share:
In this point of code the driver is:
- 1a) Removing the connection from the pool
- 1b) If the connection pool is below the minimum required connections a new connection is created -> reference
- 2a) Close the connection socket
But that flow sounds not resilient as if some error happens at the moment of creating the new connection (1b), the Socket previously removed from the pool would not be disposed (2a)
System.OperationCanceledException: The operation was canceled.
|
at Couchbase.Core.IO.Operations.OperationBase.GetResult(Int16 token) in C:\AgileContent\couchbase-net-client\src\Couchbase\Core\IO\Operations\OperationBase.cs:line 243 |
at Couchbase.Core.ClusterNode.ExecuteOp(Func`4 sender, IOperation op, Object state, CancellationTokenPair tokenPair) in C:\AgileContent\couchbase-net-client\src\Couchbase\Core\ClusterNode.cs:line 512 |
at Couchbase.Core.ClusterNode.GetErrorMap(IConnection connection, IRequestSpan span, CancellationToken cancellationToken) in C:\AgileContent\couchbase-net-client\src\Couchbase\Core\ClusterNode.cs:line 230 |
at Couchbase.Core.ClusterNode.Couchbase.Core.IO.Connections.IConnectionInitializer.InitializeConnectionAsync(IConnection connection, CancellationToken cancellationToken) in C:\AgileContent\couchbase-net-client\src\Couchbase\Core\ClusterNode.cs:line 661 |
at Couchbase.Core.IO.Connections.ConnectionPoolBase.CreateConnectionAsync(CancellationToken cancellationToken) in C:\AgileContent\couchbase-net-client\src\Couchbase\Core\IO\Connections\ConnectionPoolBase.cs:line 90 |
at Couchbase.Core.IO.Connections.Channels.ChannelConnectionPool.<>c__DisplayClass28_0.<<AddConnectionsAsync>g__StartConnection|0>d.MoveNext() in C:\AgileContent\couchbase-net-client\src\Couchbase\Core\IO\Connections\Channels\ChannelConnectionPool.cs:line 256 |
--- End of stack trace from previous location ---
|
at Couchbase.Core.IO.Connections.Channels.ChannelConnectionPool.RemoveConnectionAsync(ChannelConnectionProcessor connection) in C:\AgileContent\couchbase-net-client\src\Couchbase\Core\IO\Connections\Channels\ChannelConnectionPool.cs:line 299 |
Couchbase.Core.Exceptions.UnambiguousTimeoutException: The operation 3405/{"i":"c8253ce6bfc82c48/bfca6d6ddcc0f6ce","a":"couchbase-net-sdk/3.3.3.0 (clr/.NET 6.0.10) (os/Microsoft Windows 10.0.19045)"} timed out after 00:00:03.2604515. It was retried 0 times using Couchbase.Core.Retry.BestEffortRetryStrategy. at Couchbase.Utils.ThrowHelper.ThrowTimeoutException(IOperation operation, IErrorContext context) in C:\AgileContent\couchbase-net-client\src\Couchbase\Utils\ThrowHelper.cs:line 98 at Couchbase.Core.ClusterNode.ExecuteOp(Func`4 sender, IOperation op, Object state, CancellationTokenPair tokenPair) in C:\AgileContent\couchbase-net-client\src\Couchbase\Core\ClusterNode.cs:line 597 at Couchbase.Core.ClusterNode.Hello(IConnection connection, IRequestSpan span, CancellationToken cancellationToken) in C:\AgileContent\couchbase-net-client\src\Couchbase\Core\ClusterNode.cs:line 315 at Couchbase.Core.ClusterNode.Couchbase.Core.IO.Connections.IConnectionInitializer.InitializeConnectionAsync(IConnection connection, CancellationToken cancellationToken) in C:\AgileContent\couchbase-net-client\src\Couchbase\Core\ClusterNode.cs:line 652 at Couchbase.Core.IO.Connections.ConnectionPoolBase.CreateConnectionAsync(CancellationToken cancellationToken) in C:\AgileContent\couchbase-net-client\src\Couchbase\Core\IO\Connections\ConnectionPoolBase.cs:line 90 at Couchbase.Core.IO.Connections.Channels.ChannelConnectionPool.<>c__DisplayClass28_0.<<AddConnectionsAsync>g__StartConnection|0>d.MoveNext() in C:\AgileContent\couchbase-net-client\src\Couchbase\Core\IO\Connections\Channels\ChannelConnectionPool.cs:line 256 --- End of stack trace from previous location --- at Couchbase.Core.IO.Connections.Channels.ChannelConnectionPool.RemoveConnectionAsync(ChannelConnectionProcessor connection) in C:\AgileContent\couchbase-net-client\src\Couchbase\Core\IO\Connections\Channels\ChannelConnectionPool.cs:line 299 -----------------------Context Info--------------------------- {"dispatchedFrom":"::1","dispatchedTo":"::1","documentKey":"{\u0022i\u0022:\u0022c8253ce6bfc82c48/bfca6d6ddcc0f6ce\u0022,\u0022a\u0022:\u0022couchbase-net-sdk/3.3.3.0 (clr/.NET 6.0.10) (os/Microsoft Windows 10.0.19045)\u0022}","clientContextId":"3405","cas":0,"status":"success","bucketName":"general","collectionName":null,"scopeName":null,"message":null,"opCode":"helo","retryReasons":[]} |