[Magma] - Non-negative counter exception in setBackfillRemaining_UNLOCKED

./sequoia -client 172.23.96.162:2375 -provider file:centos_third_cluster.yml -test tests/rebalance/test_allRebalance_collections.yml -scope tests/rebalance/scope_rebalance_collection.yml -scale 3 -repeat 0 -log_level 0 -version 7.1.0-1037 -skip_setup=false -skip_test=false -skip_teardown=true -skip_cleanup=false -continue=false -collect_on_error=false -stop_on_error=false -duration=0 -show_topology=true

[2021-06-25T01:28:50-07:00, sequoiatools/couchbase-cli:7.0:c559c3] failover -c 172.23.104.232:8091 --server-failover 172.23.104.244:8091 -u Administrator -p password

[2021-06-25T01:29:23-07:00, sequoiatools/couchbase-cli:7.0:9154a2] recovery -c 172.23.104.232:8091 --server-recovery 172.23.104.244:8091 --recovery-type delta -u Administrator -p password

[2021-06-25T01:29:30-07:00, sequoiatools/couchbase-cli:7.0:3d05a0] rebalance -c 172.23.104.232:8091 -u Administrator -p password

→  

Error occurred on container - sequoiatools/couchbase-cli:7.0:[rebalance -c 172.23.104.232:8091 -u Administrator -p password]

docker logs 3d05a0

docker start 3d05a0

[root@sd1401-cnt78 logs]# grep CRITICAL memcached.log.00000*

memcached.log.000004.txt:2021-06-25T01:29:42.148735-07:00 CRITICAL *** Fatal error encountered during exception handling ***

memcached.log.000004.txt:2021-06-25T01:29:42.148789-07:00 CRITICAL Caught unhandled std::exception-derived exception. what(): ThrowExceptionUnderflowPolicy current:0 arg:-1322

memcached.log.000004.txt:2021-06-25T01:29:42.148793-07:00 CRITICAL Exception thrown from:

memcached.log.000004.txt:2021-06-25T01:29:42.148834-07:00 CRITICAL     #0  /opt/couchbase/bin/memcached() [0x400000+0x2e101e]

memcached.log.000004.txt:2021-06-25T01:29:42.148853-07:00 CRITICAL     #1  /opt/couchbase/bin/memcached() [0x400000+0xda71e]

memcached.log.000004.txt:2021-06-25T01:29:42.148899-07:00 CRITICAL     #2  /opt/couchbase/bin/memcached() [0x400000+0x479982]

memcached.log.000004.txt:2021-06-25T01:29:42.148919-07:00 CRITICAL     #3  /opt/couchbase/bin/memcached() [0x400000+0x483042]

memcached.log.000004.txt:2021-06-25T01:29:42.148938-07:00 CRITICAL     #4  /opt/couchbase/bin/memcached() [0x400000+0x484d23]

memcached.log.000004.txt:2021-06-25T01:29:42.148954-07:00 CRITICAL     #5  /opt/couchbase/bin/memcached() [0x400000+0x4868a1]

memcached.log.000004.txt:2021-06-25T01:29:42.148968-07:00 CRITICAL     #6  /opt/couchbase/bin/memcached() [0x400000+0x486e84]

memcached.log.000004.txt:2021-06-25T01:29:42.148995-07:00 CRITICAL     #7  /opt/couchbase/bin/memcached() [0x400000+0x5fdc12]

memcached.log.000004.txt:2021-06-25T01:29:42.149020-07:00 CRITICAL     #8  /opt/couchbase/bin/memcached() [0x400000+0x5fad35]

memcached.log.000004.txt:2021-06-25T01:29:42.149050-07:00 CRITICAL     #9  /opt/couchbase/bin/memcached() [0x400000+0x755d40]

memcached.log.000004.txt:2021-06-25T01:29:42.149077-07:00 CRITICAL     #10 /opt/couchbase/bin/memcached() [0x400000+0x73dafa]

memcached.log.000004.txt:2021-06-25T01:29:42.149103-07:00 CRITICAL     #11 /opt/couchbase/bin/memcached() [0x400000+0x758cf9]

memcached.log.000004.txt:2021-06-25T01:29:42.149131-07:00 CRITICAL     #12 /opt/couchbase/bin/memcached() [0x400000+0x5fa8b4]

memcached.log.000004.txt:2021-06-25T01:29:42.149192-07:00 CRITICAL     #13 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f66e4abd000+0xcdd40]

memcached.log.000004.txt:2021-06-25T01:29:42.149206-07:00 CRITICAL     #14 /lib64/libpthread.so.0() [0x7f66e699f000+0x7ea5]

memcached.log.000004.txt:2021-06-25T01:29:42.149248-07:00 CRITICAL     #15 /lib64/libc.so.6(clone+0x6d) [0x7f66e41d5000+0xfe8dd]

(gdb) bt  

#0  0x00007f66e420b387 in raise () from /lib64/libc.so.6

#1  0x00007f66e420ca78 in abort () from /lib64/libc.so.6

#2  0x00007f66e4b5663c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

#3  0x0000000000a9527b in backtrace_terminate_handler() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88

#4  0x00007f66e4b618f6 in __cxxabiv1::__terminate(void (*)()) () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

#5  0x00007f66e4b61961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

#6  0x00007f66e4b61bf4 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7f6690000940, tinfo=tinfo@entry=0xf98fc0 <typeinfo for boost::exception_detail::error_info_injector<std::underflow_error>>, 

    dest=dest@entry=0x6dd880 <boost::exception_detail::error_info_injector<std::underflow_error>::~error_info_injector()>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95

#7  0x00000000006e1132 in cb::throwWithTrace<std::underflow_error> (exception=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/exception/info.hpp:129

#8  0x00000000004da71e in cb::ThrowExceptionUnderflowPolicy<unsigned long>::underflow (current=0, arg=-1322, desired=<optimized out>, this=0x7f664f73e4f8) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:779

#9  0x0000000000879982 in store (desired=18446744073709550294, this=0x7f664f73e4f8) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:104

#10 NonNegativeCounter (initial=18446744073709550294, this=0x7f664f73e4f8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/non_negative_counter.h:81

#11 _M_construct<unsigned long&> (this=0x7f664f73e4f8) at /opt/gcc-10.2.0/include/c++/10.2.0/optional:419

#12 operator=<long unsigned int&> (__u=<synthetic pointer>, __u=<synthetic pointer>, this=0x7f664f73e4f8) at /opt/gcc-10.2.0/include/c++/10.2.0/optional:794

#13 setBackfillRemaining_UNLOCKED (value=18446744073709550294, this=0x7f664f73e390) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:642

#14 ActiveStream::setBackfillRemaining(unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:638

#15 0x0000000000883042 in DCPBackfillBySeqnoDisk::create() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill_by_seqno_disk.cc:152

#16 0x0000000000884d23 in DCPBackfillDisk::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill_disk.cc:131

#17 0x00000000008868a1 in BackfillManager::backfill() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill-manager.cc:333

#18 0x0000000000886e84 in BackfillManagerTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill-manager.cc:71

#19 0x00000000009fdc12 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68

#20 0x00000000009fad35 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f66c4fe7540)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189

#21 0x0000000000b55d40 in operator() (this=0x7f66c4fe7540) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416

#22 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (this=0x7f66e2b36000, thread=..., 

    task=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached.debug, CU 0x6463ba5, DIE 0x64a93d4>)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97

#23 0x0000000000b3dafa in folly::CPUThreadPoolExecutor::threadRun (this=0x7f66e2b36000, thread=...)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265

#24 0x0000000000b58cf9 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (

    __t=<optimized out>, __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73

#25 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>)

    at /usr/local/include/c++/7.3.0/bits/invoke.h:95

#26 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467

#27 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551

#28 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401

#29 0x00000000009fa8b4 in operator() (this=0x7f66e2ab3a80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47

#30 operator() (__closure=0x7f66e2ab3a80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47

#31 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401

#32 0x00007f66e4b8ad40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80

#33 0x00007f66e69a6ea5 in start_thread () from /lib64/libpthread.so.0

#34 0x00007f66e42d38dd in clone () from /lib64/libc.so.6