[Magma] Memcached crashed in magma::Magma::Impl::getStats during diskfull

(gdb) bt

#0  0x00007f4a393227bb in raise () from /lib/x86_64-linux-gnu/libc.so.6

#1  0x00007f4a3930d535 in abort () from /lib/x86_64-linux-gnu/libc.so.6

#2  0x00007f4a396ec7fc in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

#3  0x0000000000c82cf6 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88

#4  0x00007f4a396f7ab6 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

#5  0x00007f4a396f7b21 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

#6  0x00007f4a396f7db4 in __cxxabiv1::__cxa_throw (obj=<optimized out>, tinfo=0xe72058 <typeinfo for gsl::fail_fast>, dest=0x637a60 <gsl::fail_fast::~fail_fast()>)

    at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95

#7  0x0000000000d6b4d5 in __cxxabiv1::__cxa_throw (thrownException=0x7f4a2c004c00, type=0xe72058 <typeinfo for gsl::fail_fast>, destructor=<optimized out>)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/experimental/exception_tracer/ExceptionTracerLib.cpp:106

#8  0x0000000000461eb0 in gsl::detail::fail_fast_throw (

    message=message@entry=0xe3a830 "GSL: Precondition failure: 'diskSize >= dataSize' at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_stats.cc:175") at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/gsl-lite.hpp:1769

#9  0x00000000005169d6 in operator() (dataSize=<optimized out>, diskSize=<optimized out>, __closure=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/gsl-lite.hpp:1769

#10 magma::Magma::Impl::computeFragmentationStats (this=<optimized out>, magmaStats=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_stats.cc:181

#11 0x0000000000a975a7 in magma::Magma::Impl::getStats (this=0x7f4a14250000, magmaStats=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_stats.cc:460

#12 0x0000000000a97ed9 in magma::Magma::Impl::GetStats (cacheDuration=..., magmaStats=..., this=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_stats.cc:510

#13 magma::Magma::GetStats (this=<optimized out>, stats=..., cacheDuration=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_stats.cc:518

#14 0x00000000009cc7fd in MagmaMemoryTrackingProxy::GetStats (this=0x7f4a041cd148, cacheDuration=cacheDuration@entry=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-memory-tracking-proxy.cc:391

#15 0x000000000099f9a0 in MagmaKVStore::getAggrDbFileInfo (this=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:3880

#16 0x000000000092a086 in EPBucket::getAggregatedFileInfo (this=0x7f49bdff7000)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvshard.h:76

#17 0x0000000000930409 in EPBucket::getFileStats (this=<optimized out>, collector=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1581

#18 0x00000000007959c8 in EventuallyPersistentEngine::doEngineStatsLowCardinality (this=0x7f4a14211400, collector=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:3156

#19 0x0000000000797a76 in EventuallyPersistentEngine::doMetricGroupLow (collector=..., this=0x7f4a14211400)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:414

#20 EventuallyPersistentEngine::get_prometheus_stats (this=0x7f4a14211400, collector=..., metricGroup=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:366

#21 0x00000000006085ad in operator() (bucket=..., __closure=0x7f4a34feabc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stats.cc:316

#22 std::__invoke_impl<bool, server_prometheus_stats(const PrometheusStatCollector&, cb::prometheus::MetricGroup)::<lambda(Bucket&)>&, Bucket&> (__f=...)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60

#23 std::__invoke_r<bool, server_prometheus_stats(const PrometheusStatCollector&, cb::prometheus::MetricGroup)::<lambda(Bucket&)>&, Bucket&> (__fn=...)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:113

#24 std::_Function_handler<bool(Bucket&), server_prometheus_stats(const PrometheusStatCollector&, cb::prometheus::MetricGroup)::<lambda(Bucket&)> >::_M_invoke(const std::_Any_data &, Bucket &) (__functor=..., __args#0=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291

#25 0x0000000000630424 in std::function<bool (Bucket&)>::operator()(Bucket&) const (__args#0=..., this=0x7f4a34feabc0)

--Type <RET> for more, q to quit, c to continue without paging--

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248

#26 BucketManager::forEach(std::function<bool (Bucket&)>) (this=<optimized out>, fn=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/buckets.cc:921

#27 0x0000000000607bf3 in server_prometheus_stats (collector=..., metricGroup=cb::prometheus::MetricGroup::Low) at /opt/gcc-10.2.0/include/c++/10.2.0/new:175

#28 0x0000000000bef06f in std::function<cb::engine_errc (PrometheusStatCollector const&)>::operator()(PrometheusStatCollector const&) const (__args#0=...,

    this=0x7f4a38019820) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248

#29 cb::prometheus::MetricServer::Endpoint::Collect (this=0x7f4a380197f0)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/statistics/prometheus.cc:133

#30 0x0000000000c09746 in prometheus::detail::CollectMetrics (collectables=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/pull/src/metrics_collector.cc:20

#31 0x0000000000c07c3e in prometheus::detail::MetricsHandler::handleGet (this=0x7f4a380ff080, conn=0x2264ef0)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/pull/src/handler.cc:149

#32 0x0000000000bf51bb in CivetServer::requestHandler (conn=<optimized out>, cbdata=0x7f4a380ff080)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/CivetServer.cpp:245

#33 0x0000000000c00f70 in handle_request (conn=conn@entry=0x2264ef0)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:14288

#34 0x0000000000c02091 in handle_request_stat_log (conn=0x2264ef0)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18586

#35 process_new_connection (conn=0x2264ef0)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18586

#36 worker_thread_run (conn=0x2264ef0)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18993

#37 worker_thread (thread_func_param=0x2264ef0)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19054

#38 0x00007f4a39d23fa3 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0

#39 0x00007f4a393e3eff in clone () from /lib/x86_64-linux-gnu/libc.so.6

(gdb)

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /data/workspace/debian-p0-durability-vset00-00-steady_state_subdoc_6.5_P0_majority/testexec.11147.ini skip_cluster_reset=True,durability=MAJORITY,num_items=50000,GROUP=P0,upgrade_version=7.6.0-1455,sirius_url=http://172.23.120.103:4000 -t subdoc.sub_doc_success.BasicOps.test_basic_ops,nodes_init=2,replicas=1,num_items=10000,sub_doc_size=102400,GROUP=P0'