RejectedExecutionException when thread pool exceeded

Apr 12, 2023 1:56:03 PM com.couchbase.client.core.deps.io.grpc.internal.ManagedChannelImpl$2 uncaughtException

SEVERE: [Channel<5>: (my-grpc-route-grahamp-test2.apps.cloud-native.fg1b.p1.openshiftapps.com:443)] Uncaught exception in the SynchronizationContext. Panic!

java.util.concurrent.RejectedExecutionException: Task com.couchbase.client.core.deps.io.grpc.internal.ManagedChannelImpl$RealChannel$PendingCall$1@4a7c3f83 rejected from java.util.concurrent.ThreadPoolExecutor@7937bd82[Running, pool size = 12, active threads = 12, queued tasks = 0, completed tasks = 0]

    at java.base/java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065)

    at java.base/java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833)

    at java.base/java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365)

    at com.couchbase.client.core.deps.io.grpc.internal.ManagedChannelImpl$RealChannel$PendingCall.reprocess(ManagedChannelImpl.java:1111)

    at com.couchbase.client.core.deps.io.grpc.internal.ManagedChannelImpl$RealChannel.updateConfigSelector(ManagedChannelImpl.java:1033)

    at com.couchbase.client.core.deps.io.grpc.internal.ManagedChannelImpl$NameResolverListener$1NamesResolved.run(ManagedChannelImpl.java:1826)

    at com.couchbase.client.core.deps.io.grpc.SynchronizationContext.drain(SynchronizationContext.java:95)

    at com.couchbase.client.core.deps.io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127)

    at com.couchbase.client.core.deps.io.grpc.internal.ManagedChannelImpl$NameResolverListener.onResult(ManagedChannelImpl.java:1878)

    at com.couchbase.client.core.deps.io.grpc.internal.DnsNameResolver$Resolve.run(DnsNameResolver.java:333)

    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)

    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)

    at java.base/java.lang.Thread.run(Thread.java:833)

java.util.concurrent.RejectedExecutionException: Task com.couchbase.client.core.deps.io.grpc.internal.SerializingExecutor@52aba218 rejected from java.util.concurrent.ThreadPoolExecutor@7937bd82[Running, pool size = 11, active threads = 11, queued tasks = 0, completed tasks = 20]

    at java.base/java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065)

    at java.base/java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833)

    at java.base/java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365)

    at com.couchbase.client.core.deps.io.grpc.internal.SerializingExecutor.schedule(SerializingExecutor.java:102)

    at com.couchbase.client.core.deps.io.grpc.internal.SerializingExecutor.execute(SerializingExecutor.java:95)

    at com.couchbase.client.core.deps.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.closedInternal(ClientCallImpl.java:765)

    at com.couchbase.client.core.deps.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.closed(ClientCallImpl.java:701)

    at com.couchbase.client.core.deps.io.grpc.internal.DelayedStream$DelayedStreamListener$4.run(DelayedStream.java:510)

    at com.couchbase.client.core.deps.io.grpc.internal.DelayedStream$DelayedStreamListener.drainPendingCallbacks(DelayedStream.java:535)

    at com.couchbase.client.core.deps.io.grpc.internal.DelayedStream.drainPendingCalls(DelayedStream.java:187)

    at com.couchbase.client.core.deps.io.grpc.internal.DelayedStream.access$100(DelayedStream.java:44)

    at com.couchbase.client.core.deps.io.grpc.internal.DelayedStream$4.run(DelayedStream.java:148)

    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)

    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)

    at java.base/java.lang.Thread.run(Thread.java:833)

13:56:03.980 [Thread-4] ERROR performance.DocCreateThread - Error adding documents to cluster

com.couchbase.client.core.error.InternalServerFailureException: Internal Couchbase Server error {"completed":false,"maybeSent":true,"readonly":false,"requestName":"upsert","retried":0,"server":"Panic! This is a bug!","service":{"bucket":"default","scope":"_default","documentId":"doc_2","collection":"_default","type":"kv"},"status":"UNKNOWN","timeoutMs":30000,"timings":{"lastDispatchMicros":164850,"encodingMicros":8002,"totalDispatchMicros":164850}}

    at com.couchbase.client.core.protostellar.CoreProtostellarErrorHandlingUtil.convertKeyValueException(CoreProtostellarErrorHandlingUtil.java:113)

    at com.couchbase.client.core.protostellar.CoreProtostellarAccessors.lambda$reactive$3(CoreProtostellarAccessors.java:210)

    at com.couchbase.client.core.protostellar.CoreProtostellarAccessors$2.onFailure(CoreProtostellarAccessors.java:270)

    at com.couchbase.client.core.deps.com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1099)

    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)

    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)

    at java.base/java.lang.Thread.run(Thread.java:833)

    Suppressed: java.lang.Exception: #block terminated with an error

        at reactor.core.publisher.BlockingSingleSubscriber.blockingGet(BlockingSingleSubscriber.java:99)

        at reactor.core.publisher.Flux.blockLast(Flux.java:2746)

        at performance.DocCreateThread.run(DocCreateThread.java:58) 

    Cluster cluster = Cluster.connect("protostellar://my-grpc-route-grahamp-test2.apps.cloud-native.fg1b.p1.openshiftapps.com:443",

      ClusterOptions.clusterOptions("Administrator", "password")

        .environment(env -> env.securityConfig(sec -> {

          sec.trustManagerFactory(InsecureTrustManagerFactory.INSTANCE);

        })));

    Collection collection = cluster.bucket("default").defaultCollection();

    Flux.fromIterable(Arrays.asList(UUID.randomUUID(), UUID.randomUUID(), UUID.randomUUID(), UUID.randomUUID(), UUID.randomUUID()))

      .parallel(5)

      .runOn(Schedulers.parallel())

      .doOnNext(v -> System.out.println("sending " + v.toString()))

      .concatMap(doc -> collection.reactive().insert(doc.toString(), JsonObject.create()).doOnNext(v -> System.out.println("done " + v.toString())))

      .sequential()

      .blockLast();