================================================================= ==memcached==77953==ERROR: AddressSanitizer: heap-use-after-free on address 0x60b0000f8b88 at pc 0x000000da63d1 bp 0x7f1585968e90 sp 0x7f1585968e88 READ of size 8 at 0x60b0000f8b88 thread T13 (mc:worker_01) #0 0xda63d0 in std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::_M_find_before_node(unsigned long, cb::engine::Feature const&, unsigned long) const /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:1573 #1 0xda63d0 in std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::_M_find_node(unsigned long, cb::engine::Feature const&, unsigned long) const /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:693 #2 0xda63d0 in std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::find(cb::engine::Feature const&) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:1454 #3 0xda63d0 in std::unordered_set, std::equal_to, std::allocator >::find(cb::engine::Feature const&) /opt/gcc-10.2.0/include/c++/10.2.0/bits/unordered_set.h:650 #4 0xda63d0 in Bucket::supports(cb::engine::Feature) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/buckets.cc:51 #5 0x10fa245 in select_bucket(Cookie&, std::__cxx11::basic_string, std::allocator > const&) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/select_bucket_executor.cc:42 #6 0x10ff95e in select_bucket_executor(Cookie&) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/select_bucket_executor.cc:116 #7 0xf94c32 in std::function::operator()(Cookie&) const /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:622 #8 0xf94c32 in execute_client_request_packet(Cookie&, cb::mcbp::Request const&) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:923 #9 0xf23916 in Cookie::doExecute() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:151 #10 0xf2f576 in Cookie::execute(bool) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:164 #11 0xe4d40f in Connection::executeCommandPipeline() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:553 #12 0xe56672 in Connection::executeCommandsCallback() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:706 #13 0xe5a8b5 in Connection::rw_callback(bufferevent*, void*) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:855 #14 0x7f1590cd3d8d in bufferevent_run_deferred_callbacks_unlocked /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 #15 0x7f1590cdcd00 in event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 #16 0x7f1590cdd54e in event_process_active /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #17 0x7f1590cdd54e in event_base_loop /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #18 0x3159fb6 in folly::EventBase::loopBody(int, bool) /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #19 0x315a485 in folly::EventBase::loop() /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #20 0x315be45 in folly::EventBase::loopForever() /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #21 0xd447ec in worker_libevent /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:114 #22 0x309212a in CouchbaseThread::run() /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #23 0x309212a in platform_thread_wrap /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #24 0x7f1591540ea4 in start_thread (/lib64/libpthread.so.0+0x7ea4) #25 0x7f158e7da8dc in __clone (/lib64/libc.so.6+0xfe8dc) 0x60b0000f8b88 is located 8 bytes inside of 104-byte region [0x60b0000f8b80,0x60b0000f8be8) freed by thread T32 (NonIoPool0) here: #0 0x7f15929ee7b7 in operator delete(void*, unsigned long) (/opt/couchbase/bin/../lib/libasan.so.6+0xab7b7) #1 0xda215e in __gnu_cxx::new_allocator::deallocate(std::__detail::_Hash_node_base**, unsigned long) /opt/gcc-10.2.0/include/c++/10.2.0/ext/new_allocator.h:133 #2 0xda215e in std::allocator_traits >::deallocate(std::allocator&, std::__detail::_Hash_node_base**, unsigned long) /opt/gcc-10.2.0/include/c++/10.2.0/bits/alloc_traits.h:492 #3 0xda215e in std::__detail::_Hashtable_alloc > >::_M_deallocate_buckets(std::__detail::_Hash_node_base**, unsigned long) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable_policy.h:2099 #4 0xda215e in std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::_M_deallocate_buckets(std::__detail::_Hash_node_base**, unsigned long) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:407 #5 0xda215e in std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::_M_deallocate_buckets() /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:412 #6 0xda215e in std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::_M_move_assign(std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >&&, std::integral_constant) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:1220 #7 0xdaf1c0 in std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::operator=(std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >&&) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:519 #8 0xdaf1c0 in std::unordered_set, std::equal_to, std::allocator >::operator=(std::unordered_set, std::equal_to, std::allocator >&&) /opt/gcc-10.2.0/include/c++/10.2.0/bits/unordered_set.h:269 #9 0xdaf1c0 in BucketManager::create(Cookie&, std::__cxx11::basic_string, std::allocator >, std::__cxx11::basic_string, std::allocator >, BucketType) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/buckets.cc:249 #10 0x105dab2 in operator() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/create_remove_bucket_command_context.cc:58 #11 0x105dab2 in __invoke_impl&> /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #12 0x105dab2 in __invoke_r&> /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:110 #13 0x105dab2 in _M_invoke /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291 #14 0x104a6ae in std::function::operator()() const /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:622 #15 0x104a6ae in OneShotTask::run() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/one_shot_task.h:50 #16 0x2bd3cba in GlobalTask::execute() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #17 0x2bbf375 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #18 0x31508cf in folly::detail::function::FunctionTraits::operator()() /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #19 0x31508cf in folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 previously allocated by thread T33 (NonIoPool1) here: #0 0x7f15929ed997 in operator new(unsigned long) (/opt/couchbase/bin/../lib/libasan.so.6+0xaa997) #1 0x1301351 in __gnu_cxx::new_allocator::allocate(unsigned long, void const*) /opt/gcc-10.2.0/include/c++/10.2.0/ext/new_allocator.h:115 #2 0x1301351 in std::allocator_traits >::allocate(std::allocator&, unsigned long) /opt/gcc-10.2.0/include/c++/10.2.0/bits/alloc_traits.h:460 #3 0x1301351 in std::__detail::_Hashtable_alloc > >::_M_allocate_buckets(unsigned long) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable_policy.h:2085 #4 0x1301351 in std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::_M_allocate_buckets(unsigned long) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:398 #5 0x1301351 in std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::_M_rehash_aux(unsigned long, std::integral_constant) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:2091 #6 0x1301351 in std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::_M_rehash(unsigned long, unsigned long const&) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:2070 #7 0x1301351 in std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::_M_insert_unique_node(cb::engine::Feature const&, unsigned long, unsigned long, std::__detail::_Hash_node*, unsigned long) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:1728 #8 0x130231a in std::pair, bool> std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::_M_emplace(std::integral_constant, cb::engine::Feature&&) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:1682 #9 0x13efbdc in std::pair, bool> std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::emplace(cb::engine::Feature&&) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:788 #10 0x13efbdc in std::pair, bool> std::unordered_set, std::equal_to, std::allocator >::emplace(cb::engine::Feature&&) /opt/gcc-10.2.0/include/c++/10.2.0/bits/unordered_set.h:377 #11 0x13efbdc in EventuallyPersistentEngine::getFeatures() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:1775 #12 0xdaf1a4 in BucketManager::create(Cookie&, std::__cxx11::basic_string, std::allocator >, std::__cxx11::basic_string, std::allocator >, BucketType) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/buckets.cc:249 #13 0x105dab2 in operator() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/create_remove_bucket_command_context.cc:58 #14 0x105dab2 in __invoke_impl&> /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #15 0x105dab2 in __invoke_r&> /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:110 #16 0x105dab2 in _M_invoke /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291 #17 0x104a6ae in std::function::operator()() const /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:622 #18 0x104a6ae in OneShotTask::run() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/one_shot_task.h:50 #19 0x2bd3cba in GlobalTask::execute() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #20 0x2bbf375 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #21 0x31508cf in folly::detail::function::FunctionTraits::operator()() /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #22 0x31508cf in folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 Thread T13 (mc:worker_01) created by T0 here: #0 0x7f1592997642 in pthread_create (/opt/couchbase/bin/../lib/libasan.so.6+0x54642) #1 0x3091bb3 in cb_create_named_thread(unsigned long*, void (*)(void*), void*, int, char const*) /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:102 #2 0xd55fc0 in create_worker /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:79 #3 0xd55fc0 in worker_threads_init() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:234 #4 0xab467d in memcached_main(int, char**) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/memcached.cc:1003 #5 0x7f158e6fe554 in __libc_start_main (/lib64/libc.so.6+0x22554) Thread T32 (NonIoPool0) created by T0 here: #0 0x7f1592997642 in pthread_create (/opt/couchbase/bin/../lib/libasan.so.6+0x54642) #1 0x7f158fbfafb4 in __gthread_create /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:663 #2 0x7f158fbfafb4 in std::thread::_M_start_thread(std::unique_ptr >, void (*)()) /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:135 #3 0x2babc12 in folly::PriorityThreadFactory::newThread(folly::Function&&) /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:55 #4 0x2babc12 in CBPriorityThreadFactory::newThread(folly::Function&&) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:70 #5 0x2ba52db in CBRegisteredThreadFactory::newThread(folly::Function&&) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #6 0x315196a in folly::ThreadPoolExecutor::addThreads(unsigned long) /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:215 Thread T33 (NonIoPool1) created by T0 here: #0 0x7f1592997642 in pthread_create (/opt/couchbase/bin/../lib/libasan.so.6+0x54642) #1 0x7f158fbfafb4 in __gthread_create /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:663 #2 0x7f158fbfafb4 in std::thread::_M_start_thread(std::unique_ptr >, void (*)()) /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:135 #3 0x2babc12 in folly::PriorityThreadFactory::newThread(folly::Function&&) /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:55 #4 0x2babc12 in CBPriorityThreadFactory::newThread(folly::Function&&) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:70 #5 0x2ba52db in CBRegisteredThreadFactory::newThread(folly::Function&&) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #6 0x315196a in folly::ThreadPoolExecutor::addThreads(unsigned long) /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:215 SUMMARY: AddressSanitizer: heap-use-after-free /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:1573 in std::_Hashtable, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::_M_find_before_node(unsigned long, cb::engine::Feature const&, unsigned long) const Shadow bytes around the buggy address: 0x0c1680017120: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa 0x0c1680017130: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa 0x0c1680017140: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa 0x0c1680017150: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa 0x0c1680017160: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa =>0x0c1680017170: fa[fa]fa fa fa fa fa fa fa fa fa fa fa fa fa fa 0x0c1680017180: fa fa fa fa fa fa fd fd fd fd fd fd fd fd fd fd 0x0c1680017190: fd fd fd fd fa fa fa fa fa fa fa fa fd fd fd fd 0x0c16800171a0: fd fd fd fd fd fd fd fd fd fd fa fa fa fa fa fa 0x0c16800171b0: fa fa fd fd fd fd fd fd fd fd fd fd fd fd fd fd 0x0c16800171c0: fa fa fa fa fa fa fa fa fd fd fd fd fd fd fd fd Shadow byte legend (one shadow byte represents 8 application bytes): Addressable: 00 Partially addressable: 01 02 03 04 05 06 07 Heap left redzone: fa Freed heap region: fd Stack left redzone: f1 Stack mid redzone: f2 Stack right redzone: f3 Stack after return: f5 Stack use after scope: f8 Global redzone: f9 Global init order: f6 Poisoned by user: f7 Container overflow: fc Array cookie: ac Intra object redzone: bb ASan internal: fe Left alloca redzone: ca Right alloca redzone: cb Shadow gap: cc ==memcached==77953==ABORTING