Memcached crashed in gsl::detail::fail_fast_throw ( message=0xdfd4f0 "GSL: Precondition failure: 'se.first == SystemEvent::Collection || se.first == SystemEvent::ModifyCollection'

(gdb) bt full

#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50

        set = {__val = {0, 140649818953272, 8751390605079424880, 7954894494818989171, 0, 0, 4590863848016662724, 0, 0, 0, 0, 0, 9406202058778598705,

            7709868262426592998, 6476371942095733517, 12794242819540805634}}

        pid = <optimized out>

        tid = <optimized out>

        ret = <optimized out>

#1  0x00007febe0a89535 in __GI_abort () at abort.c:79

        save_stage = 1

        act = {__sigaction_handler = {sa_handler = 0xb, sa_sigaction = 0xb}, sa_mask = {__val = {140651064862336, 3432, 140651064844960,

              140651064862467, 140651063548063, 220, 1, 10, 140649508728048, 140649818953392, 140651044496208, 0, 140651063555657, 140651064862336,

              140651063556659, 140651064862336}}, sa_flags = -519668720, sa_restorer = 0x127c210 <stderr>}

        sigs = {__val = {32, 0 <repeats 15 times>}}

#2  0x00007febe105d63c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

        terminating = false

        t = <optimized out>

#3  0x0000000000c79f36 in backtrace_terminate_handler ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88

No locals.

#4  0x00007febe10688f6 in __cxxabiv1::__terminate (handler=<optimized out>)

    at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

No locals.

#5  0x00007febe1068961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

No locals.

#6  0x00007febe1068bf4 in __cxxabiv1::__cxa_throw (obj=<optimized out>, tinfo=0xdd2778 <typeinfo for gsl::fail_fast>, dest=

    0x631340 <gsl::fail_fast::~fail_fast()>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95

        globals = <optimized out>

        header = 0x7feb84014360

#7  0x0000000000d31435 in __cxxabiv1::__cxa_throw (thrownException=0x7feb840143e0, type=0xdd2778 <typeinfo for gsl::fail_fast>,

    destructor=<optimized out>)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/experimental/exception_tracer/ExceptionTracerLib.cpp:106

        orig_cxa_throw = 0x0

#8  0x000000000045cb9e in gsl::detail::fail_fast_throw (

    message=0xdfd4f0 "GSL: Precondition failure: 'se.first == SystemEvent::Collection || se.first == SystemEvent::ModifyCollection' at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/system"...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/gsl-lite.hpp:1769

No locals.

#9  0x00000000004b3f66 in SystemEventFactory::getCollectionIDFromKey (key=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/systemevent_factory.cc:209

        se = {<std::__pair_base<SystemEvent, cb::sized_buffer<unsigned char const> >> = {<No data fields>}, first = (unknown: -1770055536), second = {

--Type <RET> for more, q to quit, c to continue without paging--

            static npos = <optimized out>, buf = 0x7feb648cf1b3 <error: Cannot access memory at address 0x7feb648cf1b3>, len = 7}}

#10 0x00000000008624d6 in Collections::getCollectionIDFromKey (key=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/collections_types.cc:46

No locals.

#11 0x000000000088bd30 in Collections::VB::Manifest::isLogicallyDeleted (this=0x7feb76228720, key=..., seqno=8837)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_manifest.cc:992

        lookup = <optimized out>

        itr = <optimized out>

#12 0x00000000007dc0ca in non-virtual thunk to ItemPagingVisitor::visit(HashTable::HashBucketLock const&, StoredValue&) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/paging_visitor.cc:250

        aliveVBStates = {bits = {<std::_Base_bitset<1>> = {_M_w = 0}, <No data fields>}}

        NoopSeqnoAckCb = {<std::_Maybe_unary_or_binary_function<void, Vbid, long>> = {<std::binary_function<Vbid, long, void>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0,

                _M_const_object = 0x0, _M_function_pointer = 0x0, _M_member_pointer = NULL}, _M_pod_data = '\000' <repeats 15 times>},

            _M_manager = 0x0}, _M_invoker = 0x0}

        cb::NoExpiryLimit = <optimized out>

        Collections::VB::LegacyMaxSeqnoKey = <optimized out>

        Collections::DefaultCollectionIdentifier = <optimized out>

        Collections::VB::LegacyXattrKey = <optimized out>

        NoopSyncWriteCompleteCb = {<std::_Maybe_unary_or_binary_function<void, CookieIface*, cb::engine_errc>> = {<std::binary_function<CookieIface*, cb::engine_errc, void>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8,

            _M_functor = {_M_unused = {_M_object = 0x0, _M_const_object = 0x0, _M_function_pointer = 0x0, _M_member_pointer = NULL},

              _M_pod_data = '\000' <repeats 15 times>}, _M_manager = 0x0}, _M_invoker = 0x0}

        Collections::VB::LegacyJSONFormat = <optimized out>

        Collections::ManifestFile = <optimized out>

        Collections::DefaultScopeIdentifier = <optimized out>

        cb::rbac::PrivilegeAccessOk = <optimized out>

        DcpControlKeys::FlatBuffersSystemEvents = <optimized out>

        std::__ioinit = {static _S_refcount = 0, static _S_synced_with_stdio = true}

        globalBucketLoggerName = {static npos = 18446744073709551615,

          _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0},

          _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}

        Collections::ManifestFileName = <optimized out>

        cb::rbac::PrivilegeAccessFailNoPrivileges = <optimized out>

        Collections::DefaultCollectionName = <optimized out>

        cb::rbac::PrivilegeAccessFail = <optimized out>

        DcpControlKeys::ChangeStreams = <optimized out>

        Collections::NoDataLimit = <optimized out>

        Collections::CollectionEventDebugTag = <optimized out>

        Collections::DefaultScopeName = <optimized out>

--Type <RET> for more, q to quit, c to continue without paging--

        Collections::ScopeEventDebugTag = <optimized out>

#13 0x00000000007b0c02 in HashTable::pauseResumeVisit (this=0x7feb7657c618, visitor=..., start_pos=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/hash_table.cc:1328

        tmp = 0x0

        lh = {bucketNum = 426, htLock = {_M_device = 0x7feb765e1078, _M_owns = true}}

        v = <optimized out>

        paused = false

        lh = {_M_device = 0x7feb765e1000, _M_owns = false}

        vt = <optimized out>

        lock = 3

        hash_bucket = 426

#14 0x00000000007b0e1b in HashTable::visit (this=this@entry=0x7feb7657c618, visitor=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/hash_table.cc:1227

        ht_pos = {ht_size = 0, lock = 0, hash_bucket = 0}

#15 0x00000000007da7df in ItemPagingVisitor::visitBucket (vb=..., this=0x7febdd3b4d00)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/paging_visitor.cc:311

No locals.

#16 ItemPagingVisitor::visitBucket (this=0x7febdd3b4d00, vb=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/paging_visitor.cc:296

No locals.

#17 0x00000000008010cd in VBCBAdaptor::run (this=0x7feb5c34e6d0)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vb_adapters.cc:77

        debugVbid = {value = 78}

        vbid = {vbid = 78}

        vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7feb7657c600, _M_refcount = {_M_pi = 0x7feb76dc9200}}, <No data fields>}

        visitorName = {value = {_M_elems = "Item pager", '\000' <repeats 21 times>}}

#18 0x0000000000bc730d in GlobalTask::execute (this=0x7feb5c34e6d0, threadName=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:104

        guard = {previous = 0x0}

        taskableName = {value = {_M_elems = "bucket1", '\000' <repeats 24 times>}}

        executedAt = <optimized out>

        scheduleOverhead = <optimized out>

        start = <optimized out>

        runAgain = <optimized out>

        end = <optimized out>

        runtime = <optimized out>

#19 0x0000000000bc553a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7feb967f1e20)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309

        runAgain = <optimized out>

--Type <RET> for more, q to quit, c to continue without paging--

        proxy = <error reading variable>

        proxy = <optimized out>

        runAgain = <optimized out>

        _logger_ = <optimized out>

#20 folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:363

        fn = @0x7feb967f1e20: {__proxy = @0x7febdd2df680}

#21 0x0000000000bc8496 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7feb967f1e20)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

        fn = @0x7feb967f1e20: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7febdd2df680, tiny = {

              __data = "\200\366-\335\353\177", '\000' <repeats 26 times>, " \037\177\226\353\177\000\000\000\312\267\002\000\000\000",

              __align = {<No data fields>}}},

          call_ = 0xbc54b0 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>, exec_ = 0x606c00

     <folly::detail::function::DispatchSmallTrivial::exec_<16ul>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>}

#22 operator() (__closure=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

        task = {storage_ = {{emptyState = -48 '\320', value = {task = 0x7feb5c34e6d0,

                func = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7febdd2df680, tiny = {

                      __data = "\200\366-\335\353\177", '\000' <repeats 26 times>, " \037\177\226\353\177\000\000\000\312\267\002\000\000\000",

                      __align = {<No data fields>}}},

                  call_ = 0xbc54b0 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,

                  exec_ = 0x606c00 <folly::detail::function::DispatchSmallTrivial::exec_<16ul>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>}}}, hasValue = true}}

        this = <optimized out>

#23 0x0000000000d2d42f in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7feb967f20d0)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

        fn = @0x7feb967f20d0: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7febdf814800, tiny = {

              __data = "\000H\201\337\353\177\000\000\200\000\002", '\000' <repeats 13 times>, "\020\362\064\001\000\000\000\000\060\000\000\000\000\000\000\000\227\336\000\000\000\000\000", __align = {<No data fields>}}},

          call_ = 0xbc8830 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,

          exec_ = 0x606c00 <folly::detail::function::DispatchSmallTrivial::exec_<16ul>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>}

#24 folly::ThreadPoolExecutor::runTask (this=this@entry=0x7febdf814900, thread=..., task=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

        rctx = {

          prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cx--Type <RET> for more, q to quit, c to continue without paging--

x::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}

        startTime = {__d = {__r = 190969548824310}}

        stats = {expired = false, waitTime = {__r = 31724258}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 190969517100052}}, requestId = 0}

#25 0x0000000000d16127 in folly::CPUThreadPoolExecutor::threadRun (this=0x7febdf814900, thread=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:306

        task = {storage_ = {{emptyState = 0 '\000', value = {<folly::ThreadPoolExecutor::Task> = {

                  func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7febdf814800, tiny = {

                        __data = "\000H\201\337\353\177\000\000\200\000\002", '\000' <repeats 13 times>, "\020\362\064\001\000\000\000\000\060\000\000\000\000\000\000\000\227\336\000\000\000\000\000", __align = {<No data fields>}}},

                    call_ = 0xbc8830 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,

                    exec_ = 0x606c00 <folly::detail::function::DispatchSmallTrivial::exec_<16ul>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>}, enqueueTime_ = {__d = {__r = 190969517100052}}, expiration_ = {__r = 0},

                  expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x0, tiny = {

                        __data = "\000\000\000\000\000\000\000\000\227\336\000\000\000\000\000\000\227\336\000\000\000\000\000\000\tIZ", '\000' <repeats 13 times>, "\022f\017\341\353\177\000", __align = {<No data fields>}}}, call_ = 0x46bc43

     <folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0},

                  context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}, poison = false,

                priority_ = 0 '\000', queueObserverPayload_ = 140651073367605}}, hasValue = true}}

        guard = {storage_ = {{emptyState = 32 ' ', value = {list_ = {prev = 0x13d7d20 <folly::emptyList>, curr = {forbid = true,

                    allowTerminationOnBlocking = false, ex = 0x7febdf814900, tag = {static npos = <optimized out>,

                      b_ = 0x7febdf8f64e0 <error: Cannot access memory at address 0x7febdf8f64e0>,

                      e_ = 0x7febdf8f64f9 <error: Cannot access memory at address 0x7febdf8f64f9>}}}}}, hasValue = true}}

        threadIDsGuard = {<folly::detail::ScopeGuardImplBase> = {dismissed_ = false}, function_ = {__this = 0x7febdf814900}}

#26 0x0000000000d2ffca in std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__f=<optimized out>, __t=<optimized out>, __f=<optimized out>,

    __t=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100

No locals.

#27 std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:95

No locals.

#28 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (__args=..., this=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/functional:416

No locals.

#29 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() (this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/functional:499

No locals.

#30 folly::detail::function::FunctionTraits<void ()>::callSmall<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_p--Type <RET> for more, q to quit, c to continue without paging--

tr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:363

        fn = <optimized out>

#31 0x0000000000bc22d4 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7febdf911480)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

        fn = <error reading variable>

#32 CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() (__closure=0x7febdf911480)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

        threadNameOpt = {storage_ = {{emptyState = 64 '@', value = {static npos = 18446744073709551615,

                _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

                  _M_p = 0x7feb967f2240 "NonIoPool1"}, _M_string_length = 10, {_M_local_buf = "NonIoPool1\000\000\000\000\000",

                  _M_allocated_capacity = 8029725099528449870}}}, hasValue = true}}

        func = <error reading variable func (Cannot access memory at address 0x7febdf911480)>

        func = <optimized out>

        threadNameOpt = <optimized out>

#33 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:377

        fn = <error reading variable>

#34 0x00007febe1091d40 in std::execute_native_thread_routine (__p=0x7febdf9be740)

    at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80

        __t = <optimized out>

#35 0x00007febe1895fa3 in start_thread (arg=<optimized out>) at pthread_create.c:486

        ret = <optimized out>

        pd = <optimized out>

        now = <optimized out>

        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140649818998528, -1440980686950888884, 140721694226158, 140721694226159, 140649818998528, 0,

                1429812949926021708, 1429708857718475340}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0,

              cleanup = 0x0, canceltype = 0}}}

        not_first_call = <optimized out>

#36 0x00007febe0b6006f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

No locals.

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /data/workspace/debian-p0-collections-vset00-00-drop_recreate_collections_rebalance_set2_7.0_P1/testexec.7859.ini GROUP=set2,rerun=False,skip_log_scan=False,get-cbcollect-info=False,infra_log_level=critical,log_level=error,bucket_storage=couchstore,upgrade_version=7.6.0-1387,sirius_url=http://172.23.120.103:4000 -t bucket_collections.collections_drop_recreate_rebalance.CollectionsDropRecreateRebalance.test_data_load_collections_with_rebalance_in_out,nodes_init=4,nodes_in=1,nodes_out=2,bucket_spec=multi_bucket.buckets_1000_collections,GROUP=set2'