Failed during startup due to Unequal number of trees and filters

org.apache.hyracks.api.exceptions.HyracksDataException: HYR0087: Unequal number of trees and filters found in /var/cb-cache/@analytics/v_iodevice_14/storage/partition_46/Default/Default/remote_lKU8s_volCollection_3_sqhwc/0/remote_lKU8s_volCollection_3_sqhwc        at org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:57) ~[hyracks-api.jar:1.0.0-2133]        at org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager.cleanupAndGetValidFiles(LSMBTreeFileManager.java:100) ~[hyracks-storage-am-lsm-btree.jar:1.0.0-2133]        at org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex.loadDiskComponents(AbstractLSMIndex.java:206) ~[hyracks-storage-am-lsm-common.jar:1.0.0-2133]        at org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex.activate(AbstractLSMIndex.java:200) ~[hyracks-storage-am-lsm-common.jar:1.0.0-2133]        at org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.LSMColumnBTree.activate(LSMColumnBTree.java:89) ~[hyracks-storage-am-lsm-btree-column.jar:1.0.0-2133]        at org.apache.asterix.common.context.DatasetLifecycleManager.open(DatasetLifecycleManager.java:221) ~[asterix-common.jar:1.0.0-2133]        at com.couchbase.analytics.bootstrap.AnalyticsLocalRecoveryManager.cleanUp(AnalyticsLocalRecoveryManager.java:95) ~[columnar-server.jar:1.0.0-2133]        at com.couchbase.analytics.bootstrap.AnalyticsLocalRecoveryManager.startLocalRecovery(AnalyticsLocalRecoveryManager.java:58) ~[columnar-server.jar:1.0.0-2133]        at org.apache.asterix.app.nc.task.LocalRecoveryTask.perform(LocalRecoveryTask.java:45) ~[asterix-app.jar:1.0.0-2133]        at org.apache.asterix.app.replication.message.RegistrationTasksResponseMessage.handle(RegistrationTasksResponseMessage.java:63) ~[asterix-app.jar:1.0.0-2133]        at org.apache.asterix.messaging.NCMessageBroker.lambda$receivedMessage$0(NCMessageBroker.java:108) ~[asterix-app.jar:1.0.0-2133]        at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) [?:?]        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]        at java.base/java.lang.Thread.run(Thread.java:840) [?:?]

2024-06-14T14:49:26.143+00:00 INFO CBAS.util.ExitUtil [ShutdownWatchdog] starting shutdown watchdog- system will halt if shutdown is not completed within 600 seconds

2024-06-14T14:49:26.143+00:00 WARN CBAS.util.ExitUtil [JVM exit thread] JVM exiting with status 2; bye!

java.lang.Throwable: exit callstack

        at org.apache.hyracks.util.ExitUtil.exit(ExitUtil.java:92) ~[hyracks-util.jar:1.0.0-2133]

        at org.apache.asterix.app.replication.message.RegistrationTasksResponseMessage.handle(RegistrationTasksResponseMessage.java:90) ~[asterix-app.jar:1.0.0-2133]

        at org.apache.asterix.messaging.NCMessageBroker.lambda$receivedMessage$0(NCMessageBroker.java:108) ~[asterix-app.jar:1.0.0-2133]

        at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]

        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]

        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]

        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]

        at java.base/java.lang.Thread.run(Thread.java:840) ~[?:?]

2024-06-14T14:49:26.145+00:00 INFO CBAS.messaging.CCMessageBroker [Executor-9:ClusterController] Received message: NCLifecycleTaskReportMessage{nodeId='d0cafd7f4ae977394169785712bd58bf', success=false, exception=HYR0087: Unequal number of trees and filters found in /var/cb-cache/@analytics/v_iodevice_14/storage/partition_46/Default/Default/remote_lKU8s_volCollection_3_sqhwc/0/remote_lKU8s_volCollection_3_sqhwc, localCounters=null, activePartitions=[0, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31,38, 40, 46, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 89, 90, 91, 92, 93, 102, 114]}

2024-06-14T14:49:26.145+00:00 INFO CBAS.util.ExitUtil [JVM exit thread] the following (4) shutdown hooks have been registered: {Thread[Thread-6,10,main]=Thread[Thread-6,10,main], Thread[Thread-10,10,main]=Thread[Thread-10,10,main], Thread[Logging-Cleaner,5,main]=Thread[Logging-Cleaner,5,main], Thread[ShutdownHook-d0cafd7f4ae977394169785712bd58bf,5,main]=Thread[ShutdownHook-d0cafd7f4ae977394169785712bd58bf,5,main]}

sudo guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/couchbase_columnar_volume.ini -p bucket_storage=magma,bucket_eviction_policy=fullEviction,rerun=False -t aGoodDoctor.goldfish.GoldfishVolume.Columnar.test_rebalance,num_items=1000000000,num_buckets=1,bucket_names=GleamBook,bucket_type=membase,iterations=2,batch_size=1000,sdk_timeout=60,log_level=debug,infra_log_level=debug,rerun=False,skip_cleanup=True,key_size=18,randomize_doc_size=False,randomize_value=True,maxttl=10,pc=20,gsi_nodes=3,cbas_nodes=3,fts_nodes=3,kv_nodes=3,n1ql_nodes=2,kv_disk=1000,n1ql_disk=50,gsi_disk=500,fts_disk=1000,cbas_disk=1000,kv_compute=m5.4xlarge,gsi_compute=m5.4xlarge,n1ql_compute=m5.4xlarge,fts_compute=m5.4xlarge,cbas_compute=m5.4xlarge,mutation_perc=20,key_type=CircularKey,capella_run=true,services=data,rebl_services=,max_rebl_nodes=27,provider=AWS,region=us-east-1,type=GP3,size=1000,ops_rate=100000,skip_teardown_cleanup=true,wait_timeout=28800,index_timeout=28800,runtype=columnar1,skip_init=false,rebl_ops_rate=10000,collections=5,gtm=true,valType=Hotel,expiry=true,v_scaling=true,h_scaling=true,horizontal_scale=1,clients_per_db=20,track_failures=False,onPremMongo=False,num_clusters=1,remoteCouchbase=True,steady_state_workload_sleep=0,loop=2 -m rest'