[Collections] - memcached crashes in DurabilityMonitor::SyncWrite::getBySeqno

./testrunner -i /tmp/win10-bucket-ops.ini rerun=False,crash_warning=True,quota_percent=95,GROUP=durability_majority -t bucket_collections.collections_rebalance.CollectionsRebalance.test_data_load_collections_with_rebalance_in,nodes_init=3,nodes_in=2,override_spec_params=durability;replicas,durability=MAJORITY,replicas=2,bucket_spec=multi_bucket.buckets_for_rebalance_tests,data_load_stage=during,GROUP=durability_majority

(gdb) bt full

#0  0x00007f55b91a3453 in load (__m=std::memory_order_seq_cst, this=<optimized out>) at /usr/local/include/c++/7.3.0/bits/atomic_base.h:396

No locals.

#1  getBySeqno (this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/item.h:179

No locals.

#2  DurabilityMonitor::SyncWrite::getBySeqno (this=this@entry=0x7f5578176490) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/durability_monitor_impl.cc:48

No locals.

#3  0x00007f55b919c29c in ActiveDurabilityMonitor::eraseSyncWrite (this=0x7f55795eade0, key=..., seqno=seqno@entry=20)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:907

        s = {

          holder_ = {<folly::LockedPtrBase<folly::Synchronized<std::unique_ptr<ActiveDurabilityMonitor::State, std::default_delete<ActiveDurabilityMonitor::State> >, folly::SharedMutexImpl<false, void, std::atomic, false, false> >, folly::SharedMutexImpl<false, void, std::atomic, false, false>, folly::LockPolicyExclusive>> = {parent_ = 0x7f55795eae18}, static AllowsConcurrentAccess = false}}

        toErase = {_M_node = 0x7f5578176480}

        __func__ = "eraseSyncWrite"

        valid = <optimized out>

#4  0x00007f55b9269309 in VBucket::dropPendingKey (this=<optimized out>, key=..., seqno=seqno@entry=20) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:4020

No locals.

#5  0x00007f55b91af80b in EPBucket::dropKey (this=<optimized out>, vbid=..., diskKey=..., bySeqno=20, isAbort=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1093

        cHandle = {<Collections::VB::Manifest::ReadHandle> = {readLock = {lock_ = 0x7f557818a164, token_ = {type_ = folly::SharedMutexToken::INLINE_SHARED, slot_ = 32950}}, manifest = 0x7f557818a0e0}, 

          itr = {<folly::f14::detail::BaseIter<std::pair<CollectionID const, Collections::VB::ManifestEntry> const*, unsigned int>> = {<std::iterator<std::forward_iterator_tag, std::pair<CollectionID const, Collections::VB::ManifestEntry>, long, std::pair<CollectionID const, Collections::VB::ManifestEntry> const*, std::pair<CollectionID const, Collections::VB::ManifestEntry> const&>> = {<No data fields>}, <No data fields>}, current_ = 0x0, lowest_ = 0x0}, 

          key = {<DocKeyInterface<DocKey>> = {<No data fields>}, buffer = {static npos = <optimized out>, buf = 0x7f5578149fe1 <Address 0x7f5578149fe1 out of bounds>, len = 22}, encoding = Yes}}

        vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f55781b7a00, _M_refcount = {

              _M_pi = 0x7f55781134a0}}, <No data fields>}

        docKey = {<DocKeyInterface<DocKey>> = {<No data fields>}, buffer = {static npos = <optimized out>, buf = 0x7f5578149fe1 <Address 0x7f5578149fe1 out of bounds>, len = 22}, encoding = Yes}

        collectionId = <optimized out>

#6  0x00007f55b92bde18 in operator() (__args#2=<optimized out>, __args#1=<optimized out>, __args#0=..., this=0x7f556bb0c5c8) at /usr/local/include/c++/7.3.0/bits/std_function.h:706

No locals.

#7  time_purge_hook (d=0x7f556bcb3920, info=0x7f556bed8d80, item=..., ctx=0x7f556bb0c510) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:808

        diskKey = {keydata = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, 

              _M_p = 0x7f5578149fe0 <Address 0x7f5578149fe0 out of bounds>}, _M_string_length = 23, {_M_local_buf = "\027\000\000\000\000\000\000\000\230\325\266\200U\177\000", _M_allocated_capacity = 23}}}

        metadata = {_M_t = {

            _M_t = {<std::_Tuple_impl<0, MetaData*, std::default_delete<MetaData> >> = {<std::_Tuple_impl<1, std::default_delete<MetaData> >> = {<std::_Head_base<1, std::default_delete<MetaData>, true>> = {<std::default_delete<MetaData>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, MetaData*, false>> = {_M_head_impl = 0x7f556b9e07c0}, <No data fields>}, <No data fields>}}}

#8  0x00007f55b8e88715 in operator() (__args#2=..., __args#1=<optimized out>, __args#0=..., this=0x7f5580b6d948) at /usr/local/include/c++/7.3.0/bits/std_function.h:706

No locals.

#9  compact_seq_fetchcb (rq=0x7f5580b6d8a0, k=0x7f5580b6d6a0, v=0x7f5580b6d6b0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/db_compact.cc:308

        hook_action = <optimized out>

        info = 0x7f556bed8d80

        errcode = COUCHSTORE_SUCCESS

        ctx = 0x7f5580b6d920

        rawSeq = 0x7f556bda9d4c

        bpWithDeleted = <optimized out>

        bp = 17189

        ret_val = 0

        item = {buf = 0x0, size = 16777215}

#10 0x00007f55b8e73b96 in btree_lookup_inner (rq=rq@entry=0x7f5580b6d8a0, diskpos=<optimized out>, current=current@entry=0, end=end@entry=1) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_read.cc:122

        errcode_local = <optimized out>

        cmp_key = {buf = 0x7f556bda9d46 <Address 0x7f556bda9d46 out of bounds>, size = 6}

        val_buf = {buf = 0x7f556bda9d4c <Address 0x7f556bda9d4c out of bounds>, size = 66}

        next_key = <optimized out>

        bufpos = 78

        nodebuflen = 439

        errcode = <optimized out>

        nodebuf = 0x7f556bda9d40 <Address 0x7f556bda9d40 out of bounds>

#11 0x00007f55b8e73943 in btree_lookup_inner (rq=rq@entry=0x7f5580b6d8a0, diskpos=<optimized out>, current=current@entry=0, end=1) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_read.cc:76

        pointer = <optimized out>

        last_item = 1

        raw = <optimized out>

        errcode_local = <optimized out>

        cmp_key = {buf = 0x7f556bb0c646 <Address 0x7f556bb0c646 out of bounds>, size = 6}

        val_buf = {buf = 0x7f556bb0c64c <Address 0x7f556bb0c64c out of bounds>, size = 19}

---Type <return> to continue, or q <return> to quit---

        bufpos = 31

        nodebuflen = 301

        errcode = <optimized out>

        nodebuf = 0x7f556bb0c640 <Address 0x7f556bb0c640 out of bounds>

#12 0x00007f55b8e73d61 in btree_lookup (rq=rq@entry=0x7f5580b6d8a0, root_pointer=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_read.cc:162

No locals.

#13 0x00007f55b8e89ce2 in compact_seq_tree (ctx=0x7f5580b6d920, target=0x7f556bcb3920, source=0x7f556bcb3560) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/db_compact.cc:407

        srcfold = {cmp = {compare = 0x7f55b8e95e20 <seq_cmp(sized_buf const*, sized_buf const*)>}, file = 0x7f556bcb3560, num_keys = 1, fold = 1, in_fold = 1, tolerate_corruption = 0, keys = 0x7f5580b6d838, 

          callback_ctx = 0x7f5580b6d920, fetch_callback = 0x7f55b8e88640 <compact_seq_fetchcb(couchfile_lookup_request*, sized_buf const*, sized_buf const*)>, node_callback = 0x0}

        errcode = <optimized out>

        seqcmp = {compare = 0x7f55b8e95e20 <seq_cmp(sized_buf const*, sized_buf const*)>}

        low_key = {buf = 0x7f55b8ea6fa0 "", size = 6}

        low_key_list = 0x7f5580b6d860

#14 cb::couchstore::compact(_db&, char const*, unsigned long, std::function<int (_db&, _docinfo*, _sized_buf)>, std::function<int (_docinfo*&, _sized_buf)>, FileOpsInterface*) (source=..., target_filename=<optimized out>, 

    flags=flags@entry=251658242, filterCallback=..., rewriteDocInfoCallback=..., ops=ops@entry=0x7f55a82565c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/db_compact.cc:180

        CL_item = {_M_t = {

            _M_t = {<std::_Tuple_impl<0, CouchLatencyItem*, std::default_delete<CouchLatencyItem> >> = {<std::_Tuple_impl<1, std::default_delete<CouchLatencyItem> >> = {<std::_Head_base<1, std::default_delete<CouchLatencyItem>, true>> = {<std::default_delete<CouchLatencyItem>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, CouchLatencyItem*, false>> = {_M_head_impl = 0x0}, <No data fields>}, <No data fields>}}}

        CL_item_ptr = {value = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <error reading variable: No global symbol "std::atomic<CouchLatencyItem*>::is_always_lock_free".>}}

        CL_lat_collector = <optimized out>

        CL_timer = {blockTimer = {dest = 0x0, start = {__d = {__r = 0}}, name = 0x0, out = 0x0}}

        target = 0x7f556bcb3920

        tmpFile = {

          _M_elems = "/opt/couchbase/var/lib/couchbase/data/bucket1/224.couch.1.compact.btree-tmp_1\000\000\000\000\030P\000\000\000\000\000V\022\362\266U\177\000\000\364\001\000\000\000\000\000\000\340\335\266\200U\177\000\000\377\377\377\377\000\000\000\000@\344\266\200U\177\000\000\000\000\000\000\000\000\000\000Bu\354\264U\177\000\000\230z\300\250U\177\000\000\030\000\000\000@\000\000\000\000\333\266\200U\177\000\000@\332\266\200U\177\000\000\000\225\303\250U\177\000\000\200\224\303\250U\177\000\000\220\372\266\200U\177\000\000"...}

        errcode = <optimized out>

        scan_err = COUCHSTORE_SUCCESS

        ctx = {tree_writer = 0x7f55a8265400, transient_arena = 0x7f556b9e0720, persistent_arena = 0x7f556b9e0740, target_mr = 0x7f556ba0f0a0, target = 0x7f556bcb3920, 

          filterCallback = {<std::_Maybe_unary_or_binary_function<int, _db&, _docinfo*, _sized_buf>> = {<No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {

                  _M_object = 0x7f556bb0c510, _M_const_object = 0x7f556bb0c510, _M_function_pointer = 0x7f556bb0c510, 

                  _M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x7f556bb0c510, this adjustment 60}, _M_pod_data = "\020\305\260kU\177\000\000<\000\000\000\000\000\000"}, 

              _M_manager = 0x7f55b92b2060 <std::_Function_base::_Base_manager<CouchKVStore::compactDBInternal(compaction_ctx*, cb::couchstore::CompactRewriteDocInfoCallback)::<lambda(Db&, DocInfo*, sized_buf)> >::_M_manager(std::_Any_data &, const std::_Any_data &, std::_Manager_operation)>}, 

            _M_invoker = 0x7f55b92be590 <std::_Function_handler<int(_db&, _docinfo*, _sized_buf), CouchKVStore::compactDBInternal(compaction_ctx*, cb::couchstore::CompactRewriteDocInfoCallback)::<lambda(Db&, DocInfo*, sized_buf)> >::_M_invoke(const std::_Any_data &, _db &, <unknown type in /usr/lib/debug/opt/couchbase/lib/libep.so.debug, CU 0x266e4fe, DIE 0x2786e91>, <unknown type in /usr/lib/debug/opt/couchbase/lib/libep.so.debug, CU 0x266e4fe, DIE 0x2786ea1>)>}, 

          rewriteDocInfoCallback = {<std::_Maybe_unary_or_binary_function<int, _docinfo*&, _sized_buf>> = {<std::binary_function<_docinfo*&, _sized_buf, int>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {

              static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0, _M_const_object = 0x0, _M_function_pointer = 0x0, _M_member_pointer = NULL}, _M_pod_data = '\000' <repeats 15 times>}, 

              _M_manager = 0x0}, _M_invoker = 0x7f55a827e000}, flags = 251658242}

        open_flags = <optimized out>

#15 0x00007f55b92c0fa6 in CouchKVStore::compactDBInternal(compaction_ctx*, std::function<int (_docinfo*&, _sized_buf)>) (this=this@entry=0x7f55a82a9000, hook_ctx=<optimized out>, docinfo_hook=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:1066

        compactdb = {kvstore = @0x7f55a82a9000, db = 0x7f556bcb3560, fileRev = 1}

        targetDb = {kvstore = @0x7f55a82a9000, db = 0x0, fileRev = 0}

        errCode = <optimized out>

        dbfile = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f556b959fa0 <Address 0x7f556b959fa0 out of bounds>}, 

          _M_string_length = 57, {_M_local_buf = "Z\000\000\000\000\000\000\000\200\t\240\264U\177\000", _M_allocated_capacity = 90}}

        compact_file = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, 

            _M_p = 0x7f556be06900 <Address 0x7f556be06900 out of bounds>}, _M_string_length = 65, {_M_local_buf = "r\000\000\000\000\000\000\000\220\372\266\200U\177\000", _M_allocated_capacity = 114}}

        new_file = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f5580b6eb40 ""}, _M_string_length = 0, {

            _M_local_buf = '\000' <repeats 14 times>, "\360\177", _M_allocated_capacity = 0}}

        vbid = {vbid = 224}

        phosphor_internal_category_enabled_969 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <error reading variable: No global symbol "std::atomic<std::atomic<phosphor::CategoryStatus> const*>::is_always_lock_free".>}

        phosphor_internal_category_enabled_temp_969 = <optimized out>

        phosphor_internal_tpi_969 = {category = 0x2ba8a1 <Address 0x2ba8a1 out of bounds>, name = 0x2baa1c <Address 0x2baa1c out of bounds>, type = phosphor::Complete, argument_names = {_M_elems = {

              0x297bfa <Address 0x297bfa out of bounds>, 0x2c450b <Address 0x2c450b out of bounds>}}, argument_types = {_M_elems = {phosphor::is_uint, phosphor::is_none}}}

        phosphor_internal_guard_969 = {tpi = 0x7f55b961c120 <CouchKVStore::compactDBInternal(compaction_ctx*, std::function<int (_docinfo*&, _sized_buf)>)::phosphor_internal_tpi_969>, enabled = true, arg1 = 224, 

          arg2 = {<No data fields>}, start = {__d = {__r = 1181954255966572}}}

        new_rev = 2

        flags = 251658242

        vbState = <optimized out>

---Type <return> to continue, or q <return> to quit---

        localDocQueue = {<std::_Deque_base<CouchLocalDocRequest, std::allocator<CouchLocalDocRequest> >> = {

            _M_impl = {<std::allocator<CouchLocalDocRequest>> = {<__gnu_cxx::new_allocator<CouchLocalDocRequest>> = {<No data fields>}, <No data fields>}, _M_map = 0x7f5580b6dbb0, _M_map_size = 140005208415155, _M_start = {_M_cur = 0x0, 

                _M_first = 0x7f5580b6dbb0, _M_last = 0x7f55a8c3bc00, _M_node = 0x3c}, _M_finish = {_M_cur = 0x0, _M_first = 0x7f55b6f18929 <je_arena_tcache_fill_small+1225>, _M_last = 0x0, _M_node = 0x3}}}, <No data fields>}

        info = <optimized out>

        state = <optimized out>

#16 0x00007f55b92c20e9 in CouchKVStore::compactDB (this=0x7f55a82a9000, hook_ctx=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:931

        result = false

#17 0x00007f55b91b109f in EPBucket::compactInternal (this=this@entry=0x7f55a827e000, config=..., purgeSeqno=purgeSeqno@entry=0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1140

        ctx = {<std::__shared_ptr<compaction_ctx, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<compaction_ctx, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f556bb0c510, _M_refcount = {

              _M_pi = 0x7f556bb0c500}}, <No data fields>}

        shard = <optimized out>

        result = <optimized out>

        vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f55b3bdfc00, _M_refcount = {_M_pi = 0x7f55b923a4d5

     <KVShard::getBucket(Vbid) const+341>}}, <No data fields>}

#18 0x00007f55b91b14b4 in EPBucket::doCompact (this=0x7f55a827e000, config=..., purgeSeqno=0, cookie=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1208

        vb = {vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f55781b7a00, _M_refcount = {

                _M_pi = 0x7f55781134a0}}, <No data fields>}, lock = {_M_device = 0x7f5579583300, _M_owns = true}}

        err = <optimized out>

        storeProp = {efficientVBDump = StorageProperties::Yes, efficientVBDeletion = StorageProperties::Yes, persistedDeletions = StorageProperties::Yes, efficientGet = StorageProperties::Yes, concWriteCompact = StorageProperties::No, 

          byIdScan = StorageProperties::Yes}

        vbid = {vbid = 224}

#19 0x00007f55b92602d8 in CompactTask::run (this=0x7f55781da810) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:67

        phosphor_internal_category_enabled_57 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <error reading variable: No global symbol "std::atomic<std::atomic<phosphor::CategoryStatus> const*>::is_always_lock_free".>}

        phosphor_internal_category_enabled_temp_57 = <optimized out>

        phosphor_internal_tpi_57 = {category = 0x297233 <Address 0x297233 out of bounds>, name = 0x2b58b3 <Address 0x2b58b3 out of bounds>, type = phosphor::Complete, argument_names = {_M_elems = {

              0x2b58bf <Address 0x2b58bf out of bounds>, 0x2c450b <Address 0x2c450b out of bounds>}}, argument_types = {_M_elems = {phosphor::is_uint, phosphor::is_none}}}

        phosphor_internal_guard_57 = {tpi = 0x7f55b961b580 <CompactTask::run()::phosphor_internal_tpi_57>, enabled = true, arg1 = 224, arg2 = {<No data fields>}, start = {__d = {__r = 1181954255950059}}}

#20 0x00007f55b920cd63 in GlobalTask::execute (this=0x7f55781da810) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73

        guard = {previous = 0x0}

#21 0x00007f55b920358f in ExecutorThread::run (this=0x7f55b3b2fca0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:188

        curTaskDescr = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, 

            _M_p = 0x7f55b3b75ca0 <Address 0x7f55b3b75ca0 out of bounds>}, _M_string_length = 19, {_M_local_buf = "\023\000\000\000\000\000\000\000\031b\000\000\000\000\000", _M_allocated_capacity = 19}}

        woketime = <optimized out>

        scheduleOverhead = <optimized out>

        again = <optimized out>

        runtime = <optimized out>

        q = <optimized out>

        tick = 7 '\a'

        guard = {engine = 0x0}

#22 0x00007f55b7c0c777 in run (this=0x7f55b3b6a8a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58

No locals.

#23 platform_thread_wrap (arg=0x7f55b3b6a8a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71

        context = {_M_t = {

            _M_t = {<std::_Tuple_impl<0, CouchbaseThread*, std::default_delete<CouchbaseThread> >> = {<std::_Tuple_impl<1, std::default_delete<CouchbaseThread> >> = {<std::_Head_base<1, std::default_delete<CouchbaseThread>, true>> = {<std::default_delete<CouchbaseThread>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, CouchbaseThread*, false>> = {_M_head_impl = 0x7f55b3b6a8a0}, <No data fields>}, <No data fields>}}}

#24 0x00007f55b5249ea5 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#25 0x00007f55b4f728dd in clone () from /lib64/libc.so.6

No symbol table info available.

(gdb)