Unexpected server error while attempting to create bucket with half of the data nodes down

172.23.104.234 (down) 172.23.104.236 (down) || 172.23.104.235 172.23.105.191

192.168.100.91 - - [04/Mar/2024:22:05:51 -0800] "POST /pools/default/buckets HTTP/1.1" 500 44 "http://172.23.105.191:8091/ui/index.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" 11273

192.168.100.91 - - [04/Mar/2024:22:05:51 -0800] "POST /pools/default/buckets HTTP/1.1" 500 44 "http://172.23.105.191:8091/ui/index.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" 10630

192.168.100.91 - - [04/Mar/2024:22:05:51 -0800] "POST /pools/default/buckets 

[error_logger:error,2024-03-04T22:05:51.958-08:00,ns_1@172.23.105.191:<0.21282.1>:ale_error_logger_handler:do_log:101]

=========================CRASH REPORT=========================

  crasher:

    initial call: ns_orchestrator:init/1

    pid: <0.21282.1>

    registered_name: []

    exception exit: timeout

      in function  chronicle_rsm:leader_request/3 (src/chronicle_rsm.erl, line 152)

      in call from chronicle_kv:submit_command/3 (src/chronicle_kv.erl, line 673)

      in call from chronicle_kv:txn_loop_commit/8 (src/chronicle_kv.erl, line 310)

      in call from ns_bucket:do_create_bucket/4 (src/ns_bucket.erl, line 1022)

      in call from ns_bucket:create_bucket/3 (src/ns_bucket.erl, line 1007)

      in call from ns_orchestrator:idle/3 (src/ns_orchestrator.erl, line 752)

      in call from gen_statem:loop_state_callback/11 (gen_statem.erl, line 1428)

    ancestors: [ns_orchestrator_child_sup,ns_orchestrator_sup,mb_master_sup,

                  mb_master,leader_registry_sup,leader_services_sup,

                  <0.17784.1>,ns_server_sup,ns_server_nodes_sup,<0.15284.1>,

                  ns_server_cluster_sup,root_sup,<0.154.0>]

    message_queue_len: 10

    messages: [{'$gen_call',

                      {<0.16879.1132>,#Ref<0.1221501759.332922881.124448>},

                      {create_bucket,membase,"s",

                          [{num_replicas,1},

                           {replica_index,false},

                           {ram_quota,1073741824},

                           {durability_min_level,none},

                           {num_vbuckets,1024},

                           {cross_cluster_versioning_enabled,false},

                           {version_pruning_window_hrs,720},

                           {pitr_enabled,false},

                           {pitr_granularity,600},

                           {pitr_max_history_age,86400},

                           {autocompaction,false},

                           {purge_interval,undefined},

                           {flush_enabled,false},

                           {num_threads,3},

                           {eviction_policy,value_only},

                           {conflict_resolution_type,seqno},

                           {storage_mode,couchstore},

                           {max_ttl,0},

                           {compression_mode,passive},

                           {rank,0}]}},