[BP 6.6.3] connector deadlock in observed in ConcurrentRebalanceIT

"Executor-16:ClusterController" #296 daemon prio=10 os_prio=0 cpu=213.01ms elapsed=9408.05s tid=0x00007f28d009e800 nid=0x6d4 in Object.wait()  [0x00007f286d38c000]

   java.lang.Thread.State: BLOCKED (on object monitor)

	at java.lang.Object.wait(java.base@11/Native Method)

	- waiting on 

	at java.lang.Object.wait(java.base@11/Object.java:328)

	at org.apache.asterix.app.active.ActiveEntityEventsListener.waitForNonTransitionState(ActiveEntityEventsListener.java:367)

	- waiting to re-lock in wait() <0x00000007717c9558> (a com.couchbase.analytics.metadata.BucketEventsListener)

	at org.apache.asterix.app.active.ActiveEntityEventsListener.suspend(ActiveEntityEventsListener.java:581)

	- waiting to re-lock in wait() <0x00000007717c9558> (a com.couchbase.analytics.metadata.BucketEventsListener)

	at org.apache.asterix.app.active.ActiveNotificationHandler.suspendForDdlOrHalt(ActiveNotificationHandler.java:301)

	at org.apache.asterix.app.active.ActiveNotificationHandler.suspend(ActiveNotificationHandler.java:264)

	at com.couchbase.analytics.control.rebalance.Rebalance.suspendActive(Rebalance.java:243)

	at com.couchbase.analytics.control.rebalance.Rebalance.lambda$doRebalance$2(Rebalance.java:197)

	at com.couchbase.analytics.control.rebalance.Rebalance$$Lambda$920/0x0000000800a10c40.run(Unknown Source)

	at org.apache.hyracks.api.util.InvokeUtil.runUninterruptible(InvokeUtil.java:235)

	at com.couchbase.analytics.control.rebalance.Rebalance.lambda$doRebalance$3(Rebalance.java:197)

	at com.couchbase.analytics.control.rebalance.Rebalance$$Lambda$664/0x000000080073c440.run(Unknown Source)

	at org.apache.hyracks.api.util.InvokeUtil.tryWithCleanups(InvokeUtil.java:196)

	at com.couchbase.analytics.control.rebalance.Rebalance.doRebalance(Rebalance.java:195)

	at com.couchbase.analytics.control.rebalance.Rebalance.doCall(Rebalance.java:152)

	at com.couchbase.analytics.control.rebalance.Rebalance.doCall(Rebalance.java:80)

	at com.couchbase.analytics.runtime.WriteLockCallable.call(WriteLockCallable.java:27)

	at java.util.concurrent.FutureTask.run(java.base@11/FutureTask.java:264)

	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11/ThreadPoolExecutor.java:1128)

	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11/ThreadPoolExecutor.java:628)

	at java.lang.Thread.run(java.base@11/Thread.java:834)

"Executor-28:ClusterController" #385 daemon prio=10 os_prio=0 cpu=166.17ms elapsed=9013.73s tid=0x00007f28d006d800 nid=0x1c96 waiting on condition  [0x00007f28841e5000]

   java.lang.Thread.State: WAITING (parking)

	at jdk.internal.misc.Unsafe.park(java.base@11/Native Method)

	- parking to wait for  <0x0000000760d092a0> (a java.util.concurrent.locks.ReentrantReadWriteLock$FairSync)

	at java.util.concurrent.locks.LockSupport.park(java.base@11/LockSupport.java:194)

	at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11/AbstractQueuedSynchronizer.java:885)

	at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(java.base@11/AbstractQueuedSynchronizer.java:917)

	at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@11/AbstractQueuedSynchronizer.java:1240)

	at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(java.base@11/ReentrantReadWriteLock.java:959)

	at org.apache.asterix.metadata.lock.DatasetLock.modifyWriteLock(DatasetLock.java:160)

	at org.apache.asterix.metadata.lock.DatasetLock.lock(DatasetLock.java:200)

	at org.apache.asterix.common.metadata.LockList.add(LockList.java:52)

	at org.apache.asterix.metadata.lock.MetadataLockManager.acquireDatasetExclusiveModificationLock(MetadataLockManager.java:93)

	at com.couchbase.analytics.lang.ConnectLinkStatement.bucketGoneOnConnect(ConnectLinkStatement.java:612)

	at com.couchbase.analytics.lang.ConnectLinkStatement.connect(ConnectLinkStatement.java:570)

	at com.couchbase.analytics.metadata.BucketEventsListener.fix(BucketEventsListener.java:250)

	- locked <0x00000007717c9558> (a com.couchbase.analytics.metadata.BucketEventsListener)

	at com.couchbase.analytics.metadata.BucketEventsListener.lambda$handleFixEvent$1(BucketEventsListener.java:149)

	at com.couchbase.analytics.metadata.BucketEventsListener$$Lambda$1013/0x0000000800ad8040.run(Unknown Source)

	at com.couchbase.analytics.metadata.BucketEventsListener.reconnect(BucketEventsListener.java:176)

	at com.couchbase.analytics.metadata.BucketEventsListener.lambda$submitReconnect$2(BucketEventsListener.java:158)

	at com.couchbase.analytics.metadata.BucketEventsListener$$Lambda$1014/0x0000000800ad8440.call(Unknown Source)

	at java.util.concurrent.FutureTask.run(java.base@11/FutureTask.java:264)

	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11/ThreadPoolExecutor.java:1128)

	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11/ThreadPoolExecutor.java:628)

	at java.lang.Thread.run(java.base@11/Thread.java:834)

2021-04-01T17:55:55.329-07:00 INFO ClusterIntegrationUtil [Rebalancer-1] Rebalancing from [0, 3] to [0]

2021-04-01T17:55:55.341-07:00 INFO ClusterIntegrationUtil [Rebalancer-1] ensuring joining nodes ([]) are fully removed

2021-04-01T17:55:55.341-07:00 INFO ClusterIntegrationUtil [Rebalancer-1] ensured joining nodes are fully removed (failures: [])

2021-04-01T17:55:55.341-07:00 INFO ClusterIntegrationUtil [Rebalancer-1]   removing nodes [3]

2021-04-01T17:55:55.341-07:00 INFO ClusterIntegrationUtil [Rebalancer-1]   calling rebalance with following args: [rebalance, -c, 172.18.0.3:9000, -u, couchbase, -p, couchbase, --server-remove, 172.18.0.3:9003]

2021-04-01T17:55:55.341-07:00 INFO ClusterIntegrationUtil [Rebalancer-1] Running cli: rebalance -c 172.18.0.3:9000 -u couchbase -p couchbase --server-remove 172.18.0.3:9003

2021-04-01T17:55:55.782-07:00 INFO KvStoreHttpHelper [main] Bucket (gbook_users) deleted successfully.

2021-04-01T17:55:55.788-07:00 INFO KvStoreHttpHelper [main] Bucket (gbook_users) does not exist.

2021-04-01T17:55:55.819-07:00 INFO KvStoreHttpHelper [main] Got '{"_":"Cannot create buckets during rebalance"}'; will retry up to 120s (sleeping 2s and trying again)...

2021-04-01T17:55:57.826-07:00 INFO KvStoreHttpHelper [main] Got '{"_":"Cannot create buckets during rebalance"}'; will retry up to 120s (sleeping 2s and trying again)...

2021-04-01T17:55:59.832-07:00 INFO KvStoreHttpHelper [main] Got '{"_":"Cannot create buckets during rebalance"}'; will retry up to 120s (sleeping 2s and trying again)...

2021-04-01T17:56:01.838-07:00 INFO KvStoreHttpHelper [main] Got '{"_":"Cannot create buckets during rebalance"}'; will retry up to 120s (sleeping 2s and trying again)...

2021-04-01T17:56:03.844-07:00 INFO KvStoreHttpHelper [main] Got '{"_":"Cannot create buckets during rebalance"}'; will retry up to 120s (sleeping 2s and trying again)...

2021-04-01T17:56:05.850-07:00 INFO KvStoreHttpHelper [main] Got '{"_":"Cannot create buckets during rebalance"}'; will retry up to 120s (sleeping 2s and trying again)...

2021-04-01T17:56:07.856-07:00 INFO KvStoreHttpHelper [main] Got '{"_":"Cannot create buckets during rebalance"}'; will retry up to 120s (sleeping 2s and trying again)...