Scale down/rebalance out is failing in a loop with a lot of connection errors.

2024-06-06T14:52:22.323+00:00 WARN CBAS.buffercache.BufferCache [SAO:JID:0.4472:TAID:TID:ANID:ODID:6:0:17:0] Failed to read page. Retrying attempt (4/5)org.apache.hyracks.api.exceptions.HyracksDataException: HYR0094: Cannot read closed file (/var/cb-cache/@analytics/v_iodevice_1/storage/partition_17/Default/Default/remote_6GWbx_volCollection_0_ekwhd/0/remote_6GWbx_volCollection_0_ekwhd/0_664_b)        at org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:66) ~[hyracks-api.jar:1.0.0-2126]        at org.apache.hyracks.control.nc.io.IOManager.doSyncRead(IOManager.java:312) ~[hyracks-control-nc.jar:1.0.0-2126]        at org.apache.hyracks.control.nc.io.IoRequest.handle(IoRequest.java:121) ~[hyracks-control-nc.jar:1.0.0-2126]        at org.apache.hyracks.control.nc.io.IoRequestHandler.run(IoRequestHandler.java:58) ~[hyracks-control-nc.jar:1.0.0-2126]        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]        at java.base/java.lang.Thread.run(Thread.java:840) [?:?]

Caused by: java.nio.channels.ClosedChannelException        at java.base/sun.nio.ch.FileChannelImpl.ensureOpen(FileChannelImpl.java:159) ~[?:?]        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:814) ~[?:?]        at org.apache.hyracks.control.nc.io.IOManager.doSyncRead(IOManager.java:297) ~[hyracks-control-nc.jar:1.0.0-2126]        ... 5 more

2024-06-06T14:52:22.408+00:00 INFO CBAS.work.AbortTasksWork [Worker:17642ff647474026c59bcc5ea03b4d12] Aborting Tasks: JID:0.4472:[TAID:TID:ANID:ODID:6:0:7:0, TAID:TID:ANID:ODID:6:0:17:0, TAID:TID:ANID:ODID:6:0:74:0, TAID:TID:ANID:ODID:6:0:78:0, TAID:TID:ANID:ODID:6:0:82:0, TAID:TID:ANID:ODID:6:0:86:0, TAID:TID:ANID:ODID:6:0:90:0, TAID:TID:ANID:ODID:6:0:94:0, TAID:TID:ANID:ODID:6:0:98:0, TAID:TID:ANID:ODID:6:0:102:0, TAID:TID:ANID:ODID:6:0:106:0, TAID:TID:ANID:ODID:6:0:110:0, TAID:TID:ANID:ODID:6:0:114:0, TAID:TID:ANID:ODID:6:0:118:0, TAID:TID:ANID:ODID:6:0:122:0, TAID:TID:ANID:ODID:6:0:126:0]2024-06-06T14:52:22.409+00:00 WARN CBAS.util.ResourceReleaseUtils [SAO:JID:0.4472:TAID:TID:ANID:ODID:6:0:102:0] Failure closing a closeable resourcejava.lang.IllegalMonitorStateException: attempt to unlock read lock, not locked by current thread        at java.base/java.util.concurrent.locks.ReentrantReadWriteLock$Sync.unmatchedUnlockException(ReentrantReadWriteLock.java:448) ~[?:?]

        at java.base/java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:432) ~[?:?]        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1094) ~[?:?]

        at java.base/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:897) ~[?:?]        at org.apache.hyracks.cloud.buffercache.page.CloudCachedPage.afterRead(CloudCachedPage.java:44) ~[hyracks-cloud.jar:1.0.0-2126]        at org.apache.hyracks.storage.am.lsm.btree.column.cloud.buffercache.read.CloudColumnReadContext.onUnpin(CloudColumnReadContext.java:92) ~[hyracks-storage-am-lsm-btree-column.jar:1.0.0-2126]        at org.apache.hyracks.storage.common.buffercache.BufferCache.unpin(BufferCache.java:607) ~[hyracks-storage-common.jar:1.0.0-2126]        at org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeRangeSearchCursor.releasePages(ColumnBTreeRangeSearchCursor.java:190) ~[hyracks-storage-am-lsm-btree-column.jar:1.0.0-2126]

        at org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeRangeSearchCursor.doClose(ColumnBTreeRangeSearchCursor.java:263) ~[hyracks-storage-am-lsm-btree-column.jar:1.0.0-2126]        at org.apache.hyracks.storage.common.EnforcedIndexCursor.close(EnforcedIndexCursor.java:118) ~[hyracks-storage-common.jar:1.0.0-2126]        at org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor.doClose(LSMIndexSearchCursor.java:131) ~[hyracks-storage-am-lsm-common.jar:1.0.0-2126]        at org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeRangeSearchCursor.doClose(LSMBTreeRangeSearchCursor.java:71) ~[hyracks-storage-am-lsm-btree.jar:1.0.0-2126]        at org.apache.hyracks.storage.common.EnforcedIndexCursor.close(EnforcedIndexCursor.java:118) ~[hyracks-storage-common.jar:1.0.0-2126]        at org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeSearchCursor.doClose(LSMBTreeSearchCursor.java:95) ~[hyracks-storage-am-lsm-btree.jar:1.0.0-2126]        at org.apache.hyracks.storage.common.EnforcedIndexCursor.close(EnforcedIndexCursor.java:118) ~[hyracks-storage-common.jar:1.0.0-2126]

        at org.apache.hyracks.storage.am.common.util.ResourceReleaseUtils.close(ResourceReleaseUtils.java:50) ~[hyracks-storage-am-common.jar:1.0.0-2126]        at org.apache.hyracks.storage.am.common.dataflow.IndexSearchOperatorNodePushable.releaseResources(IndexSearchOperatorNodePushable.java:367) ~[hyracks-storage-am-common.jar:1.0.0-2126]        at org.apache.hyracks.storage.am.common.dataflow.IndexSearchOperatorNodePushable.close(IndexSearchOperatorNodePushable.java:331) ~[hyracks-storage-am-common.jar:1.0.0-2126]        at org.apache.hyracks.algebricks.runtime.operators.std.EmptyTupleSourceRuntimeFactory$1.close(EmptyTupleSourceRuntimeFactory.java:61) ~[algebricks-runtime.jar:1.0.0-2126]

        at org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor$SourcePushRuntime.initialize(AlgebricksMetaOperatorDescriptor.java:181) ~[algebricks-runtime.jar:1.0.0-2126]

        at org.apache.hyracks.api.rewriter.runtime.SuperActivityOperatorNodePushable.lambda$runInParallel$0(SuperActivityOperatorNodePushable.java:233) ~[hyracks-api.jar:1.0.0-2126]

        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]

        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]

        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]

        at java.base/java.lang.Thread.run(Thread.java:840) [?:?]

2024-06-06T14:53:27.607+00:00 ERRO CBAS.tcp.TCPEndpoint [TCPEndpoint IO Thread [/0.0.0.0:9116]] Unexpected tcp io error in connection TCPConnection[Remote Address: svc-da-node-005.cbqpbprse50eob-a.sandbox.nonprod-project-avengers.com/10.0.0.252:9116 Local Address: /0.0.0.0:9116]org.apache.hyracks.api.exceptions.NetException: Socket Closed        at org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.driveReaderStateMachine(MultiplexedConnection.java:360) ~[hyracks-net.jar:1.0.0-2126]        at org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.notifyIOReady(MultiplexedConnection.java:119) ~[hyracks-net.jar:1.0.0-2126]        at org.apache.hyracks.net.protocols.tcp.TCPEndpoint$IOThread.run(TCPEndpoint.java:199) [hyracks-net.jar:1.0.0-2126]2024-06-06T14:53:27.607+00:00 ERRO CBAS.tcp.TCPEndpoint [TCPEndpoint IO Thread [/0.0.0.0:9117]] Unexpected tcp io error in connection TCPConnection[Remote Address: /10.0.3.182:59456 Local Address: /0.0.0.0:9117]org.apache.hyracks.api.exceptions.NetException: Socket Closed

        at org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.driveReaderStateMachine(MultiplexedConnection.java:360) ~[hyracks-net.jar:1.0.0-2126]        at org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.notifyIOReady(MultiplexedConnection.java:119) ~[hyracks-net.jar:1.0.0-2126]