[System test]: Analytics rebalance failed with java.lang.InterruptedException

Analytics unable to successfully rebalance a34947d34ff042839b9cdded1d53ed02 due to 'java.lang.InterruptedException'; see analytics log for detailsanalytics 000ns_1@172.23.104.23 12:01:39 AM   Thu Aug 9, 2018

2018-08-09T00:01:39.172-07:00 INFO CBAS.active.ActiveNotificationHandler [Executor-259:ClusterController] Default.Local.other-3(CouchbaseMetadataExtension) resumed

2018-08-09T00:01:39.172-07:00 INFO CBAS.rebalance.Rebalance [Executor-259:ClusterController] active events handler resumed

2018-08-09T00:01:39.244-07:00 ERRO CBAS.rebalance.Rebalance [Executor-259:ClusterController] rebalance failed

java.lang.InterruptedException: null

at java.lang.Object.wait(Native Method) ~[?:1.8.0_181]

at java.lang.Object.wait(Object.java:502) ~[?:1.8.0_181]

at org.apache.hyracks.ipc.api.RPCInterface$Request.getResponse(RPCInterface.java:88) ~[hyracks-ipc.jar:6.0.0-1466]

at org.apache.hyracks.ipc.api.RPCInterface.call(RPCInterface.java:39) ~[hyracks-ipc.jar:6.0.0-1466]

at org.apache.hyracks.api.client.HyracksClientInterfaceRemoteProxy.waitForCompletion(HyracksClientInterfaceRemoteProxy.java:125) ~[hyracks-api.jar:6.0.0-1466]

at org.apache.hyracks.api.client.HyracksConnection.waitForCompletion(HyracksConnection.java:181) ~[hyracks-api.jar:6.0.0-1466]

at org.apache.asterix.common.utils.JobUtils.runJob(JobUtils.java:49) ~[asterix-common.jar:6.0.0-1466]

at org.apache.asterix.common.utils.JobUtils.runJob(JobUtils.java:38) ~[asterix-common.jar:6.0.0-1466]

at org.apache.asterix.utils.RebalanceUtil.populateDataToRebalanceTarget(RebalanceUtil.java:323) ~[asterix-app.jar:6.0.0-1466]

at org.apache.asterix.utils.RebalanceUtil.rebalance(RebalanceUtil.java:226) ~[asterix-app.jar:6.0.0-1466]

at org.apache.asterix.utils.RebalanceUtil.rebalance(RebalanceUtil.java:131) ~[asterix-app.jar:6.0.0-1466]

at com.couchbase.analytics.control.rebalance.Rebalance.rebalanceDataset(Rebalance.java:426) ~[cbas-server.jar:6.0.0-1466]

at com.couchbase.analytics.control.rebalance.Rebalance.rebalanceDatasets(Rebalance.java:251) ~[cbas-server.jar:6.0.0-1466]

at com.couchbase.analytics.control.rebalance.Rebalance.lambda$doRebalance$3(Rebalance.java:183) ~[cbas-server.jar:6.0.0-1466]

at org.apache.hyracks.api.util.InvokeUtil.tryWithCleanups(InvokeUtil.java:191) ~[hyracks-api.jar:6.0.0-1466]

at com.couchbase.analytics.control.rebalance.Rebalance.doRebalance(Rebalance.java:179) ~[cbas-server.jar:6.0.0-1466]

at com.couchbase.analytics.control.rebalance.Rebalance.doCall(Rebalance.java:139) [cbas-server.jar:6.0.0-1466]

at com.couchbase.analytics.control.rebalance.Rebalance.doCall(Rebalance.java:74) [cbas-server.jar:6.0.0-1466]

at com.couchbase.analytics.runtime.WriteLockCallable.call(WriteLockCallable.java:21) [cbas-connector.jar:6.0.0-1466]

at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_181]

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_181]

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_181]

at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181]

2018-08-09T00:01:39.364-07:00 INFO CBAS.rebalance.Rebalance [Executor-259:ClusterController] %%%%%%%%%%%%% exit rebalance {"id":"a34947d34ff042839b9cdded1d53ed02","currentTopologyRev":null,"type":"topology-change-rebalance","keepNodes":[{"nodeInfo":{"nodeId":"c5908b83453dee35fa98faabfb226306","priority":0,"opaque":{"cc-http-port":"9111","controller-id":"0","host":"172.23.104.21","num-iodevices":"2","starting-partition-id":"0"}},"recoveryType":"recovery-full"},{"nodeInfo":{"nodeId":"b903869763ab411f1f85c34f03fd6405","priority":0,"opaque":{"cc-http-port":"9111","controller-id":"2","host":"172.23.104.25","num-iodevices":"2","starting-partition-id":"4"}},"recoveryType":"recovery-full"},{"nodeInfo":{"nodeId":"93244e458e7d97671916631deb66ffc9","priority":0,"opaque":{"cc-http-port":"9111","controller-id":"3","host":"172.23.104.93","num-iodevices":"2","starting-partition-id":"6"}},"recoveryType":"recovery-full"},{"nodeInfo":{"nodeId":"5bc264ce8de125fd5dc6fa854a98dd0b","priority":0,"opaque":{"cc-http-port":"9111","controller-id":"4","host":"172.23.104.94","num-iodevices":"2","starting-partition-id":"8"}},"recoveryType":"recovery-full"},{"nodeInfo":{"nodeId":"5203592a9eba459955d1a495128b7784","priority":0,"opaque":{"cc-http-port":"9111","controller-id":"5","host":"172.23.96.96","num-iodevices":"2","starting-partition-id":"10"}},"recoveryType":"recovery-full"}],"ejectNodes":[{"nodeId":"50b4188e1490f01347eb5867cd43fc16","priority":0,"opaque":{"cc-http-port":"9111","controller-id":"1","host":"172.23.104.23","num-iodevices":"2","starting-partition-id":"2"}}]}

2018-08-09T00:01:39.365-07:00 WARN CBAS.rebalance.Rebalance [HttpExecutor(port:9111)-14] Rebalance (a34947d34ff042839b9cdded1d53ed02) cancelled successfully