Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-47231

Heap use-after-free on Bucket::supports during bucket delete

    XMLWordPrintable

Details

    • Untriaged
    • 1
    • Yes
    • KV-Engine Sprint 2021 July

    Description

      As seen during investigation of MB-47139 on node 172.23.121.135 with ASan build:

      memcached==77953==ERROR: AddressSanitizer: heap-use-after-free on address 0x60b0000f8b88 at pc 0x000000da63d1 bp 0x7f1585968e90 sp 0x7f1585968e88
      READ of size 8 at 0x60b0000f8b88 thread T13 (mc:worker_01)
          #0 0xda63d0 in std::_Hashtable<cb::engine::Feature, cb::engine::Feature, std::allocator<cb::engine::Feature>, std::__detail::_Identity, std::equal_to<cb::engine::Feature>, std::hash<cb::engine::Feature>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true> >::_M_find_before_node(unsigned long, cb::engine::Feature const&, unsigned long) const /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:1573
          #1 0xda63d0 in std::_Hashtable<cb::engine::Feature, cb::engine::Feature, std::allocator<cb::engine::Feature>, std::__detail::_Identity, std::equal_to<cb::engine::Feature>, std::hash<cb::engine::Feature>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true> >::_M_find_node(unsigned long, cb::engine::Feature const&, unsigned long) const /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:693
          #2 0xda63d0 in std::_Hashtable<cb::engine::Feature, cb::engine::Feature, std::allocator<cb::engine::Feature>, std::__detail::_Identity, std::equal_to<cb::engine::Feature>, std::hash<cb::engine::Feature>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true> >::find(cb::engine::Feature const&) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:1454
          #3 0xda63d0 in std::unordered_set<cb::engine::Feature, std::hash<cb::engine::Feature>, std::equal_to<cb::engine::Feature>, std::allocator<cb::engine::Feature> >::find(cb::engine::Feature const&) /opt/gcc-10.2.0/include/c++/10.2.0/bits/unordered_set.h:650
          #4 0xda63d0 in Bucket::supports(cb::engine::Feature) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/buckets.cc:51
          #5 0x10fa245 in select_bucket(Cookie&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/select_bucket_executor.cc:42
          #6 0x10ff95e in select_bucket_executor(Cookie&) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/select_bucket_executor.cc:116
          #7 0xf94c32 in std::function<void (Cookie&)>::operator()(Cookie&) const /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:622
          #8 0xf94c32 in execute_client_request_packet(Cookie&, cb::mcbp::Request const&) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:923
          #9 0xf23916 in Cookie::doExecute() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:151
          #10 0xf2f576 in Cookie::execute(bool) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:164
          #11 0xe4d40f in Connection::executeCommandPipeline() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:553
          #12 0xe56672 in Connection::executeCommandsCallback() /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:706
          #13 0xe5a8b5 in Connection::rw_callback(bufferevent*, void*) /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:855
      @
          #14 0x7f1590cd3d8d in bufferevent_run_deferred_callbacks_unlocked /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208
          #15 0x7f1590cdcd00 in event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726
          #16 0x7f1590cdd54e in event_process_active /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789
          #17 0x7f1590cdd54e in event_base_loop /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012
          #18 0x3159fb6 in folly::EventBase::loopBody(int, bool) /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397
          #19 0x315a485 in folly::EventBase::loop() /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315
          #20 0x315be45 in folly::EventBase::loopForever() /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538
          #21 0xd447ec in worker_libevent /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:114
          #22 0x309212a in CouchbaseThread::run() /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51
          #23 0x309212a in platform_thread_wrap /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64
          #24 0x7f1591540ea4 in start_thread (/lib64/libpthread.so.0+0x7ea4)
          #25 0x7f158e7da8dc in __clone (/lib64/libc.so.6+0xfe8dc)
       
      0x60b0000f8b88 is located 8 bytes inside of 104-byte region [0x60b0000f8b80,0x60b0000f8be8)
      freed by thread T32 (NonIoPool0) here:
          #0 0x7f15929ee7b7 in operator delete(void*, unsigned long) (/opt/couchbase/bin/../lib/libasan.so.6+0xab7b7)
          #1 0xda215e in __gnu_cxx::new_allocator<std::__detail::_Hash_node_base*>::deallocate(std::__detail::_Hash_node_base**, unsigned long) /opt/gcc-10.2.0/include/c++/10.2.0/ext/new_allocator.h:133
          #2 0xda215e in std::allocator_traits<std::allocator<std::__detail::_Hash_node_base*> >::deallocate(std::allocator<std::__detail::_Hash_node_base*>&, std::__detail::_Hash_node_base**, unsigned long) /opt/gcc-10.2.0/include/c++/10.2.0/bits/alloc_traits.h:492
          #3 0xda215e in std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<cb::engine::Feature, true> > >::_M_deallocate_buckets(std::__detail::_Hash_node_base**, unsigned long) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable_policy.h:2099
          #4 0xda215e in std::_Hashtable<cb::engine::Feature, cb::engine::Feature, std::allocator<cb::engine::Feature>, std::__detail::_Identity, std::equal_to<cb::engine::Feature>, std::hash<cb::engine::Feature>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true> >::_M_deallocate_buckets(std::__detail::_Hash_node_base**, unsigned long) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:407
          #5 0xda215e in std::_Hashtable<cb::engine::Feature, cb::engine::Feature, std::allocator<cb::engine::Feature>, std::__detail::_Identity, std::equal_to<cb::engine::Feature>, std::hash<cb::engine::Feature>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true> >::_M_deallocate_buckets() /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:412
          #6 0xda215e in std::_Hashtable<cb::engine::Feature, cb::engine::Feature, std::allocator<cb::engine::Feature>, std::__detail::_Identity, std::equal_to<cb::engine::Feature>, std::hash<cb::engine::Feature>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true> >::_M_move_assign(std::_Hashtable<cb::engine::Feature, cb::engine::Feature, std::allocator<cb::engine::Feature>, std::__detail::_Identity, std::equal_to<cb::engine::Feature>, std::hash<cb::engine::Feature>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true> >&&, std::integral_constant<bool, true>) /opt/gcc-10.2.0/include/c++/10.2.0/bits/hashtable.h:1220
      

      Complete ASan output attached: sanitizers.log.memcached.77953

      Test run Logs: https://cb-jira.s3.us-east-2.amazonaws.com/kv_log/test_2.tar.gz

      Attachments

        Issue Links

          Activity

            People

              trond Trond Norbye
              drigby Dave Rigby (Inactive)
              Votes:
              0 Vote for this issue
              Watchers:
              4 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved:

                PagerDuty