2022-03-06T11:54:09.830406-08:00 INFO <none>: Delete bucket [default]. Notifying engine

2022-03-06T11:54:09.837324-08:00 INFO <none>: Delete bucket [default]. Engine ready for shutdown

2022-03-06T11:54:09.837327-08:00 INFO <none>: Delete bucket [default]. Wait for 6 clients to disconnect

2022-03-06T11:54:09.838533-08:00 INFO <none>: Delete bucket [default]. Shut down the bucket

2022-03-06T11:54:14.502380-08:00 INFO (default) Deleted KvBucket.

2022-03-06T11:54:16.056152-08:00 INFO <none>: Delete bucket [default]. Clean up allocated resources

2022-03-06T11:54:16.056200-08:00 INFO <none>: Delete bucket [default] complete

2022-03-06T11:55:30.125015-08:00 INFO 96: Create Couchbase bucket [default]

2022-03-06T11:55:30.147945-08:00 INFO (default) Warmup::processCreateVBucketsComplete unblocking 1 cookie(s)

2022-03-06T11:55:30.148367-08:00 INFO (default) Warmup completed: 0 keys and 0 values loaded in 4231 us (0 keys/s), mem_used now at 4.588072 MB (1084.349639130889 MB/s)

bash-5.1$ grep "Bucket \"default\" not found during ensure_bucket" ns_server.debug.log  | grep "2022-03-06T12"  | wc -l

       0

bash-5.1$ grep "Bucket \"default\" not found during ensure_bucket" ns_server.debug.log  | grep "2022-03-06T13"  | wc -l

    3710

bash-5.1$ grep "Bucket \"default\" not found during ensure_bucket" ns_server.debug.log  | grep "2022-03-06T14"  | wc -l

   15766

bash-5.1$ grep "Bucket \"default\" not found during ensure_bucket" ns_server.debug.log  | grep "2022-03-06T15"  | wc -l

   14758

bash-5.1$ grep "Bucket \"default\" not found during ensure_bucket" ns_server.debug.log  | grep "2022-03-06T16"  | wc -l

   14758

bash-5.1$ grep "Bucket \"default\" not found during ensure_bucket" ns_server.debug.log  | grep "2022-03-06T17"  | wc -l

   14632

bash-5.1$ grep "Bucket \"default\" not found during ensure_bucket" ns_server.debug.log  | grep "2022-03-06T18"  | wc -l

   13969

bash-5.1$ grep "Bucket \"default\" not found during ensure_bucket" ns_server.debug.log  | grep "2022-03-06T19" | wc -l

   13680

bash-5.1$ grep "Bucket \"default\" not found during ensure_bucket" ns_server.debug.log  | grep "2022-03-06T20" | wc -l

   13425

bash-5.1$ grep "Bucket \"default\" not found during ensure_bucket" ns_server.debug.log  | grep "2022-03-06T21" | wc -l

    5716

2022-03-06T11:55:31.106216-08:00 INFO (default) EventuallyPersistentEngine::enableTraffic() result true

2022-03-06T11:55:31.106224-08:00 INFO (default) EventuallyPersistentEngine::handleTrafficControlCmd() Data traffic to persistence engine is enabled

[ns_server:error,2022-03-06T21:25:34.012-08:00,ns_1@172.23.100.35:<0.8764.47>:ns_server_stats:report_prom_stats:172]ns_server stats reporting exception: error:badarg

[{ets,lookup,

      [ns_server_stats,{c,{<<"rest_request_enters">>,[]}}],

      [{error_info,#{cause => id,module => erl_stdlib_errors}}]},

 {ns_server_stats,'-report_ns_server_lc_stats/1-fun-0-',2,

                  [{file,"src/ns_server_stats.erl"},{line,257}]},

 {lists,foreach,2,[{file,"lists.erl"},{line,1342}]},

 {ns_server_stats,'-report_prom_stats/2-fun-0-',2,

                  [{file,"src/ns_server_stats.erl"},{line,170}]},

 {ns_server_stats,report_prom_stats,2,

                  [{file,"src/ns_server_stats.erl"},{line,180}]},

 {async,'-async_init/4-fun-1-',3,[{file,"src/async.erl"},{line,191}]}]

[ns_server:error,2022-03-06T21:25:34.012-08:00,ns_1@172.23.100.35:<0.8764.47>:ns_server_stats:report_prom_stats:172]system stats reporting exception: exit:{noproc,

                                        {gen_server,call,

                                         [ns_server_stats,get_stats]}}

[{gen_server,call,2,[{file,"gen_server.erl"},{line,239}]},

 {ns_server_stats,report_system_stats,1,

                  [{file,"src/ns_server_stats.erl"},{line,188}]},

 {ns_server_stats,'-report_prom_stats/2-fun-0-',2,

                  [{file,"src/ns_server_stats.erl"},{line,170}]},

 {ns_server_stats,report_prom_stats,2,

                  [{file,"src/ns_server_stats.erl"},{line,182}]},

 {async,'-async_init/4-fun-1-',3,[{file,"src/async.erl"},{line,191}]}]

[ns_server:info,2022-03-06T11:54:09.160-08:00,ns_1@172.23.100.35:ns_memcached-default<0.1374.23>:ns_memcached:delete_bucket:876]Deleting bucket "default" from memcached (force = true)

[ns_server:error,2022-03-06T11:54:09.163-08:00,ns_1@172.23.100.35:ns_memcached-default<0.1374.23>:ns_memcached:delete_bucket:883]Failed to delete bucket "default": {error,{badmatch,{error,closed}}}

2022-03-06T12:56:13.974-08:00, failover:0:info:message(ns_1@172.23.105.164) - Starting failing over ['ns_1@172.23.100.35']

2022-03-06T12:56:13.974-08:00, ns_orchestrator:0:info:message(ns_1@172.23.105.164) - Starting failover of nodes ['ns_1@172.23.100.35']. Operation Id = 29f343f24f75cfd315eb971c4e193d8d

2022-03-06T12:56:14.311-08:00, ns_memcached:0:info:message(ns_1@172.23.100.35) - Shutting down bucket "bucket1" on 'ns_1@172.23.100.35' for deletion

2022-03-06T12:56:14.455-08:00, ns_memcached:0:info:message(ns_1@172.23.100.35) - Shutting down bucket "bucket2" on 'ns_1@172.23.100.35' for deletion

2022-03-06T12:56:14.590-08:00, failover:0:info:message(ns_1@172.23.105.164) - Failed over ['ns_1@172.23.100.35']: ok

2022-03-06T12:56:14.596-08:00, failover:0:info:message(ns_1@172.23.105.164) - Deactivating failed over nodes ['ns_1@172.23.100.35']

2022-03-06T12:56:14.762-08:00, ns_orchestrator:0:info:message(ns_1@172.23.105.164) - Failover completed successfully.

[ns_server:error,2022-03-06T12:56:44.618-08:00,ns_1@172.23.100.35:<0.32666.16>:timeout_diag_logger:do_diag:123]

{<0.3068.0>,

 [{backtrace,[<<"Program counter: 0x00007efd8184c578 (prim_inet:recv0/3 + 240)">>,

              <<>>,<<"0x00007efcc046b8b0 53">>,<<>>,

              <<"0x00007efcc046b8b8 #Port<0.201>">>,<<>>,

              <<"0x00007efcc046b8c0 Return addr 0x00007efccae86f2c (mc_binary:recv/3 + 228)">>,

              <<"y(0)     infinity">>,<<"y(1)     res">>,

              <<"y(2)     #Port<0.201>">>,<<>>,

              <<"0x00007efcc046b8e0 Return addr 0x00007efccae7b300 (mc_client_binary:cmd_vocal_recv/5 + 144)">>,

              <<"y(0)     []">>,<<"y(1)     undefined">>,

              <<"y(2)     undefined">>,<<"y(3)     134">>,<<>>,

              <<"0x00007efcc046b908 Return addr 0x00007efccae7c1f4 (mc_client_binary:delete_bucket/3 + 420)">>,

              <<"y(0)     []">>,<<"y(1)     []">>,<<>>,

              <<"0x00007efcc046b920 Return addr 0x00007efccacc612c (ns_memcached:delete_bucket/4 + 372)">>,

              <<"y(0)     false">>,<<"y(1)     []">>,

              <<"y(2)     \"bucket2\"">>,<<"y(3)     []">>,

              <<"y(4)     Catch 0x00007efccacc6157 (ns_memcached:delete_bucket/4 + 415)">>,

              <<"y(5)     Catch 0x00007efccacc62a4 (ns_memcached:delete_bucket/4 + 748)">>,

              <<>>,

              <<"0x00007efcc046b958 Return addr 0x00007efccacc5a80 (ns_memcached:terminate/2 + 232)">>,

              <<"y(0)     []">>,<<"y(1)     []">>,<<"y(2)     \"bucket2\"">>,

              <<"y(3)     Catch 0x00007efccacc5b3f (ns_memcached:terminate/2 + 423)">>,

              <<>>,

...

  {messages,[<<"{'$gen_call',{<0.31320.16>,[alias|#Ref<0.1760518760.1410138119.215166>]},warmed}">>,

             <<"check_config">>,

             <<"{'$gen_call',{<0.31792.16>,[alias|#Ref<0.1760518760.1409875976.139592>]},warmed}">>,

             <<"{'$gen_call',{<0.32230.16>,[alias|#Ref<0.1760518760.1411448833.201399>]},warmed}">>]},

 {dictionary,[{'$initial_call',{ns_memcached,init,1}},

               {'$ancestors',[<0.3067.0>,'single_bucket_kv_sup-bucket2',

                              ns_bucket_sup,ns_bucket_worker_sup,

                              ns_server_sup,ns_server_nodes_sup,<0.1752.0>,

                              ns_server_cluster_sup,root_sup,<0.145.0>]}]},

  {registered_name,'ns_memcached-bucket2'},

2022-03-06T13:42:49.784-08:00, ns_orchestrator:0:info:message(ns_1@172.23.105.164) - Starting rebalance, KeepNodes = ['ns_1@172.23.105.206','ns_1@172.23.106.177',

                                 'ns_1@172.23.100.35','ns_1@172.23.105.164'], EjectNodes = [], Failed over and being ejected nodes = [], Delta recovery nodes = ['ns_1@172.23.100.35'],  Delta recovery b\

uckets = all; Operation Id = e3e3a85f0e65f39a1b9eae473faeeed4

2022-03-06T13:44:49.789-08:00, ns_rebalancer:0:critical:message(ns_1@172.23.105.164) - Failed to wait deletion of some buckets on some nodes: [{'ns_1@172.23.100.35',

                                                         {'EXIT',

                                                          {old_buckets_shutdown_wait_failed,

                                                           ["default",

                                                            "bucket2"]}}}]

# date; curl -s localhost:8091/diag/eval -u Administrator:password -d "Pid = whereis('ns_memcached-bucket2')"

Mon Mar  7 05:25:03 PST 2022

<0.3068.0>

...

# date; curl -s localhost:8091/diag/eval -u Administrator:password -d "Pid = whereis('ns_memcached-bucket2'), erlang:process_info(Pid, backtrace)"

Mon Mar  7 05:23:30 PST 2022

{backtrace,<<"Program counter: 0x00007efd8184c578 (prim_inet:recv0/3 + 240)\n\n0x00007efcc046b8b0 53\n\n0x00007efcc046b8b8 #Port<0.201>\n\n0x00007efcc046b8c0 Return addr 0x00007efccae86f2c (mc_binary:recv/3 + 228)\ny(0)     infinity\ny(1)     res\ny(2)     #Port<0.201>\n\n0x00007efcc046b8e0 Return addr 0x00007efccae7b300 (mc_client_binary:cmd_vocal_recv/5 + 144)\ny(0)     []\ny(1)     undefined\ny(2)     undefined\ny(3)     134\n\n0x00007efcc046b908 Return addr 0x00007efccae7c1f4 (mc_client_binary:delete_bucket/3 + 420)\ny(0)     []\ny(1)     []\n\n0x00007efcc046b920 Return addr 0x00007efccacc612c (ns_memcached:delete_bucket/4 + 372)\ny(0)     false\ny(1)     []\ny(2)     \"bucket2\"\ny(3)     []\ny(4)     Catch 0x00007efccacc6157 (ns_memcached:delete_bucket/4 + 415)\ny(5)     Catch 0x00007efccacc62a4 (ns_memcached:delete_bucket/4 + 748)\n\n0x00007efcc046b958 Return addr 0x00007efccacc5a80 (ns_memcached:terminate/2 + 232)\ny(0)     []\ny(1)     []\ny(2)     \"bucket2\"\ny(3)     Catch 0x00007efccacc5b3f (ns_memcached:terminate/2 + 423)\n\n0x00007efcc046b980 Return addr 0x00007efd81a120f4 (gen_server:try_terminate/3 + 332)\ny(0)     []\ny(1)     []\ny(2)     []\ny(3)     Catch 0x00007efd81a1214a (gen_server:try_terminate/3 + 418)\n\n0x00007efcc046b9a8 Return addr 0x00007efd81a14738 (gen_server:terminate/10 + 192)\ny(0)     []\ny(1)     []\ny(2)     []\ny(3)     {state,0,0,0,{[],[]},{[],[]},{[],[]},warmed,{1646,596522,559857},\"bucket2\",[collections,json],#Port<0.201>,[{<0.3102.0>,#Ref<0.1760518760.1411383297.200086>},{<0.3099.0>,#Ref<0.1760518760.1409548295.56458>},{<0.3101.0>,#Ref<0.1760518760.1409548293.57702>},{<0.3100.0>,#Ref<0.1760518760.1409548295.56455>}],[],<0.3069.0>,false,10000}\ny(4)     ns_memcached\ny(5)     {'EXIT',<0.3067.0>,shutdown}\ny(6)     undefined\ny(7)     'ns_memcached-bucket2'\ny(8)     shutdown\ny(9)     [{gen_server,decode_msg,9,[{file,\"gen_server.erl\"},{line,481}]},{proc_lib,init_p_do_apply,3,[{file,\"proc_lib.erl\"},{line,226}]}]\ny(10)    shutdown\ny(11)    exit\n\n0x00007efcc046ba10 Return addr 0x00007efd819416d0 (proc_lib:init_p_do_apply/3 + 192)\ny(0)     []\ny(1)     []\ny(2)     Catch 0x00007efd819416f0 (proc_lib:init_p_do_apply/3 + 224)\n\n0x00007efcc046ba30 Return addr 0x00007efd818308c4 (unknown function)\n\n0x00007efcc046ba38 Return addr 0x00007efd818308c8 (<terminate process normally>)\n">>}

2022-03-06T12:56:14.313614-08:00 INFO 2663: Delete bucket [bucket1]. Wait for 254 clients to disconnect

2022-03-06T12:56:14.405602-08:00 INFO 2663: Delete bucket [bucket1] complete

2022-03-06T12:56:14.456964-08:00 INFO 67: Delete bucket [bucket2].

2022-03-06T12:56:14.455819-08:00 INFO 67: Delete bucket [bucket2]. Notifying engine

2022-03-06T12:56:14.455836-08:00 INFO (bucket2) Shutting down all DCP connections in preparation for bucket deletion.

2022-03-06T12:56:14.455840-08:00 INFO (bucket2) Shutting down dcp connections!

2022-03-06T12:56:14.455885-08:00 INFO 1433: (bucket2) DCP (Producer) eq_dcpq:replication:ns_1@172.23.100.35->ns_1@172.23.106.177:bucket2 - Destroying connection. Created 3646 s ago. Paused 18501 times, for 1h:0m:46s total. Details: {"Initializing": {"count":1, "duration":"55 us"},"ReadyListEmpty": {"count":18500, "duration":"1h:0m:46s"}}

2022-03-06T12:56:14.455934-08:00 INFO 1438: (bucket2) DCP (Producer) eq_dcpq:replication:ns_1@172.23.100.35->ns_1@172.23.105.206:bucket2 - Destroying connection. Created 3646 s ago. Paused 17701 times, for 1h:0m:46s total. Details: {"Initializing": {"count":1, "duration":"36 us"},"ReadyListEmpty": {"count":17700, "duration":"1h:0m:46s"}}

2022-03-06T12:56:14.455977-08:00 INFO 1436: (bucket2) DCP (Producer) eq_dcpq:replication:ns_1@172.23.100.35->ns_1@172.23.105.164:bucket2 - Destroying connection. Created 3646 s ago. Paused 16684 times, for 1h:0m:46s total. Details: {"Initializing": {"count":1, "duration":"37 us"},"ReadyListEmpty": {"count":16683, "duration":"1h:0m:46s"}}

2022-03-06T12:56:14.456010-08:00 INFO 1442: (bucket2) DCP (Consumer) eq_dcpq:replication:ns_1@172.23.106.177->ns_1@172.23.100.35:bucket2 - Destroying connection. Created 3646 s ago. Paused 13323 times, for 1h:0m:46s total. Details: {"ReadyListEmpty": {"count":13323, "duration":"1h:0m:46s"}}

2022-03-06T12:56:14.456038-08:00 INFO 1432: (bucket2) DCP (Consumer) eq_dcpq:replication:ns_1@172.23.105.206->ns_1@172.23.100.35:bucket2 - Destroying connection. Created 3646 s ago. Paused 13312 times, for 1h:0m:46s total. Details: {"ReadyListEmpty": {"count":13312, "duration":"1h:0m:46s"}}

2022-03-06T12:56:14.456060-08:00 INFO 1405: (bucket2) DCP (Consumer) eq_dcpq:replication:ns_1@172.23.105.164->ns_1@172.23.100.35:bucket2 - Destroying connection. Created 3646 s ago. Paused 13763 times, for 1h:0m:47s total. Details: {"ReadyListEmpty": {"count":13763, "duration":"1h:0m:47s"}}

2022-03-06T12:56:14.456946-08:00 INFO 67: Delete bucket [bucket2]. Engine ready for shutdown

2022-03-06T12:56:14.456964-08:00 INFO 67: Delete bucket [bucket2]. Wait for 255 clients to disconnect

2022-03-06T12:56:14.461984-08:00 INFO 67: Delete bucket [bucket2]. Shut down the bucket

2022-03-06T12:56:14.461999-08:00 INFO (bucket2) Shutting down dcp connections!

2022-03-06T12:56:14.462005-08:00 INFO (bucket2) Attempting to stop flusher:0

2022-03-06T12:56:14.462054-08:00 INFO (bucket2) Attempting to stop flusher:1

2022-03-06T12:56:14.462057-08:00 INFO (bucket2) Attempting to stop flusher:2

2022-03-06T12:56:14.462062-08:00 INFO (bucket2) Attempting to stop flusher:3

2022-03-06T12:56:14.462131-08:00 INFO (bucket2) Attempting to stop flusher:4

2022-03-06T12:56:14.462135-08:00 INFO (bucket2) Attempting to stop flusher:5

2022-03-06T12:56:14.462140-08:00 INFO (bucket2) Attempting to stop flusher:6

2022-03-06T12:56:14.462146-08:00 INFO (bucket2) Attempting to stop flusher:7

2022-03-06T12:56:14.462161-08:00 INFO (bucket2) Stopping bg fetchers

2022-03-06T12:56:14.462760-08:00 INFO (bucket2) KVBucket::deinitialize forceShutdown:true

2022-03-06T12:56:14.475861-08:00 INFO (bucket2) MagmaKVStore: 0 deinitializing

2022-03-06T12:56:14.475878-08:00 INFO [(bucket2) magma_0]Closing magma db (thread pool refcount 16)

2022-03-06T12:56:14.478263-08:00 INFO [(bucket2) magma_0/kvstore-0/rev-000000001]KVStore::Shutdown

..

2022-03-06T12:56:14.664401-08:00 INFO [(bucket2) magma_0]Completed closing magma db

2022-03-06T12:56:14.665002-08:00 INFO (bucket2) MagmaKVStore: 0 deinitialized

2022-03-06T12:56:14.665011-08:00 INFO (bucket2) MagmaKVStore: 1 deinitializing

2022-03-06T12:56:14.665017-08:00 INFO [(bucket2) magma_1]Closing magma db (thread pool refcount 15)

2022-03-06T12:56:14.666930-08:00 INFO [(bucket2) magma_1/kvstore-1/rev-000000001]KVStore::Shutdown

..

2022-03-06T12:56:14.929878-08:00 INFO [(bucket2) magma_2]Completed closing magma db

2022-03-06T12:56:14.930431-08:00 INFO (bucket2) MagmaKVStore: 2 deinitialized

2022-03-06T12:56:14.930446-08:00 INFO (bucket2) MagmaKVStore: 3 deinitializing

2022-03-06T12:56:14.930450-08:00 INFO [(bucket2) magma_3]Closing magma db (thread pool refcount 13)

2022-03-06T12:56:14.932092-08:00 INFO [(bucket2) magma_3/kvstore-3/rev-000000001]KVStore::Shutdown

INFO [(bucket2) magma_3]Completed closing magma db

INFO (bucket2) MagmaKVStore: 3 deinitialized

magma/db.cc:        LOG_INFO("Completed closing magma db");

..

2022-03-06T12:56:14.929567-08:00 INFO [(bucket2) magma_2/kvstore-850/rev-000000001]KVStore::Shutdown

2022-03-06T12:56:14.929878-08:00 INFO [(bucket2) magma_2]Completed closing magma db

2022-03-06T12:56:14.930431-08:00 INFO (bucket2) MagmaKVStore: 2 deinitialized

2022-03-06T12:56:14.930446-08:00 INFO (bucket2) MagmaKVStore: 3 deinitializing

2022-03-06T12:56:14.930450-08:00 INFO [(bucket2) magma_3]Closing magma db (thread pool refcount 13)

2022-03-06T12:56:14.932092-08:00 INFO [(bucket2) magma_3/kvstore-3/rev-000000001]KVStore::Shutdown

2022-03-06T12:56:14.933891-08:00 INFO [(bucket2) magma_3/kvstore-11/rev-000000001]KVStore::Shutdown

2022-03-06T12:56:14.935874-08:00 INFO [(bucket2) magma_3/kvstore-19/rev-000000001]KVStore::Shutdown

2022-03-06T12:56:14.937778-08:00 INFO [(bucket2) magma_3/kvstore-27/rev-000000001]KVStore::Shutdown

..

2022-03-06T12:56:15.054037-08:00 INFO [(bucket2) magma_3/kvstore-851/rev-000000001]KVStore::Shutdown

Thread 25 (Thread 0x7f9e53fff700 (LWP 123256)):

#0  0x00007f9ec7cb9a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

#1  0x00007f9ec5ea88bc in __gthread_cond_wait (__mutex=<optimized out>, __cond=__cond@entry=0x7f9ec01d8ba0)

    at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865

#2  std::condition_variable::wait (this=this@entry=0x7f9ec01d8ba0, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53

#3  0x000000000098bfcc in magma::WaitGroup::Wait (this=this@entry=0x7f9ec01d8b68) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/common.cc:74

#4  0x000000000091e681 in magma::Magma::Impl::Close() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:204

#5  0x000000000091e8c8 in magma::Magma::Close (this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:216

#6  0x0000000000887c1b in MagmaMemoryTrackingProxy::Close() ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-memory-tracking-proxy.cc:113

#7  0x0000000000865d81 in MagmaKVStore::deinitialize (this=0x7f9ec016b400)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:734

#8  0x000000000081a586 in operator() (shard=..., __closure=<synthetic pointer>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvshard.h:72

#9  forEachShard<EPBucket::deinitialize()::<lambda(KVShard&)> > (p=..., this=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucketmap.h:148

#10 EPBucket::deinitialize (this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:333

#11 0x00000000006a30d2 in EventuallyPersistentEngine::destroyInner (this=0x7f9ec00f8000, force=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:2256

#12 0x00000000006a3162 in EventuallyPersistentEngine::destroy(bool) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:207

#13 0x00000000005c4eca in operator() (engine=<optimized out>, this=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/engine.h:793

#14 reset (__p=0x0, this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_ptr.h:182

#15 reset (__p=0x0, this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_ptr.h:456

#16 destroyEngine (force=true, this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/buckets.cc:62

#17 BucketManager::destroy(Cookie*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/buckets.cc:436

#18 0x000000000062cbbe in operator() (__closure=0x7f9e07b91420) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/char_traits.h:322

#19 __invoke_impl<void, CreateRemoveBucketCommandContext::remove()::<lambda()>&> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60

#20 __invoke_r<void, CreateRemoveBucketCommandContext::remove()::<lambda()>&> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:110

#21 std::_Function_handler<void (), CreateRemoveBucketCommandContext::remove()::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291

#22 0x000000000062b204 in operator() (this=0x7f9eb81b1608) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248

#23 OneShotTask::run (this=0x7f9eb81b1590) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/one_shot_task.h:50

#24 0x0000000000a9fb89 in GlobalTask::execute(std::basic_string_view<char, std::char_traits<char> >) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98

#25 0x0000000000a9927a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f9e53fed650)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309

#26 0x0000000000aa0f3e in operator() (this=0x7f9e53fed650)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

#27 CancellableCPUExecutor::add(GlobalTask*, folly::Function<void ()>)::{lambda()#1}::operator()() const ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

#28 0x0000000000bf5a40 in operator() (this=0x7f9e53fed840)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416

#29 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f9ec3f51d00, 

    thread=..., task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-2440.x86_64.debug, CU 0xa389644, DIE 0xa40d579>)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97

#30 0x0000000000be04da in folly::CPUThreadPoolExecutor::threadRun (this=0x7f9ec3f51d00, thread=...)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265

#31 0x0000000000bf89f9 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73

#32 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:95

#33 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467

#34 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551

#35 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401

#36 0x0000000000a98f74 in operator() (this=0x7f9ec43e1d40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

#37 operator() (__closure=0x7f9ec43e1d40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

#38 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401

#39 0x00007f9ec5eadd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80

#40 0x00007f9ec7cb5ea5 in start_thread () from /lib64/libpthread.so.0

#41 0x00007f9ec55f69fd in clone () from /lib64/libc.so.6

void EPBucket::deinitialize() {

    stopFlusher();

    stopBgFetcher();

    KVBucket::deinitialize();

    // Perform a snapshot of the stats before shutting down so we can

    // persist the type of shutdown (stats.forceShutdown), and consequently

    // on the next warmup can determine is there was a clean shutdown - see

    // Warmup::cleanShutdown. This must be called after KVBucket::deinitialize()

    // to prevent a race condition with the StatSnap task as the deinitialize

    // function will cancel all running tasks.

    snapshotStats(true /*shuttingDown*/);

    // Now that we've stopped all of our tasks, stop any tasks the storage

    // layer may have created.

    vbMap.forEachShard([](KVShard& shard) {

        shard.getRWUnderlying()->deinitialize();

    });

}

void MagmaKVStore::deinitialize() {

    logger->info("MagmaKVStore: {} deinitializing", configuration.getShardId());

    // Close shuts down all of the magma background threads (compaction is the

    // one that we care about here). The compaction callbacks require the magma

    // instance to exist so we must do this before we reset it.

    magma->Close();

    // Flusher should have already been stopped so it should be safe to destroy

    // the magma instance now

    magma.reset();

    logger->info("MagmaKVStore: {} deinitialized", configuration.getShardId());

}

void Magma::Impl::Close() {

    if (opened) {

        LOG_INFO("Closing magma db (thread pool refcount {})",

                 threadPool.use_count());

        heartbeatWg.Deactivate();

        heartbeatToken.Cancel();

        if (threadPool) {

            threadPool->CancelTasks(threadPoolGroupId);

        }

        heartbeatWg.Wait();

        kvstores.Shutdown(cfg.CheckpointOnShutdown, wal.get());

        kvstoreWg.Wait();                      <--- !!

        opened = false;

        LOG_INFO("Completed closing magma db");

    }

    {

        ExecutionEnvGuard envGuard(

                ExecutionEnvRegistry::GetDefaultExecutionEnv());

        threadPool.reset();

    }

}

KVStore::KVStore(KVStoreConfig& config,

                 Magma::KVStoreID kvID,

                 Magma::KVStoreRevision kvsRev)

    : cfg(config),

      kvID(kvID),

      kvsRev(kvsRev),

      purgeChecker(cfg.GetExpiryFragThreshold(), cfg.GetTimeNow),

      remover(config.GetConfig().FS,

              config.GetThreadPool(),

              config.GetThreadPoolGroupId(),

              config.GetConfig().ExecutionEnv,

              fileRemoverWg) {

..

    cfg.GetWaitGroup().Add(1);

}

KVStore::~KVStore() {

    // Wait for any pending flushes

    flusherWg.Deactivate();

    flusherWg.Wait();

    currSnapshot.reset();

    currDiskSnapshot.reset();

    keyIndex.reset();

    seqIndex.reset();

    localIndex.reset();

    fileRemoverWg.Deactivate();

    fileRemoverWg.Wait();

    {

        auto cb = destroyCallback.lock();

        if (*cb) {

            LOG_INFO("KVStore::~KVStore Destroying {}", GetPath());

            (*cb)(GetPath());

        }

    }

    cfg.GetWaitGroup().Add(-1);

}

# grep 'KVStore::Shutdown' /opt/couchbase/var/lib/couchbase/logs/memcached.log.000021.txt | grep bucket2 | grep magma_3  |wc -l

65