Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-50194

[NexusKVStore] Memcached aborted in NexusKVStore::listPersistedVbuckets() ()

    XMLWordPrintable

Details

    Description

      Steps to repro:

      1. Create a 1 node cluster(172.23.122.245 (bucket_ram_quota = 2056 MB/node), replicas=0)
      2. Create 2 non default scopes (scope1 & scope2)
      3. Create 5 non default collections in all the scopes including default scope
      4. Start Loading 5M docs in each of non default collections(all scopes)
      5. While doc load is going on tried adding a node which was already part of a cluster(Prepare Join failed) and Memcached aborted in NexusKVStore::listPersistedVbuckets() ()

      *BackTrace:

      #0  0x00007fd9bb5da337 in raise () from /lib64/libc.so.6
      No symbol table info available.
      #1  0x00007fd9bb5dba28 in abort () from /lib64/libc.so.6
      No symbol table info available.
      #2  0x00000000006d43d2 in cb::handleError(spdlog::logger&, spdlog::level::level_enum, std::basic_string_view<char, std::char_traits<char> >, cb::ErrorHandlingMethod) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/error_handler.cc:43
      No locals.
      #3  0x00000000008fb401 in handleError (msg=..., this=0x7fd9a46dc5c0)
          at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/nexus-kvstore/nexus-kvstore-config.h:39
      No locals.
      #4  NexusKVStore::listPersistedVbuckets() ()
          at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/nexus-kvstore/nexus-kvstore.cc:892
              msg = {static npos = 18446744073709551615,
                _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
                  _M_p = 0x7fd916749c00 <Address 0x7fd916749c00 out of bounds>}, _M_string_length = 1104, {
                  _M_local_buf = "P\004\000\000\000\000\000\000&v\000\000\000\000\000", _M_allocated_capacity = 1104}}
              vbid = {vbid = 0}
              i = 0
              primaryVbStates = {<std::_Vector_base<vbucket_state*, std::allocator<vbucket_state*> >> = {
                  _M_impl = {<std::allocator<vbucket_state*>> = {<__gnu_cxx::new_allocator<vbucket_state*>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<vbucket_state*, std::allocator<vbucket_state*> >::_Vector_impl_data> = {_M_start = 0x7fd8c3811400, _M_finish = 0x7fd8c3811800,
                      _M_end_of_storage = 0x7fd8c3811800}, <No data fields>}}, <No data fields>}
              secondaryVbStates = {<std::_Vector_base<vbucket_state*, std::allocator<vbucket_state*> >> = {
                  _M_impl = {<std::allocator<vbucket_state*>> = {<__gnu_cxx::new_allocator<vbucket_state*>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<vbucket_state*, std::allocator<vbucket_state*> >::_Vector_impl_data> = {_M_start = 0x7fd91f2f6400, _M_finish = 0x7fd91f2f6800,
                      _M_end_of_storage = 0x7fd91f2f6800}, <No data fields>}}, <No data fields>}
      #5  0x000000000074521d in Warmup::populateShardVbStates() ()
          at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvshard.h:72
              kvStoreVbStates = {<std::_Vector_base<vbucket_state*, std::allocator<vbucket_state*> >> = {
                  _M_impl = {<std::allocator<vbucket_state*>> = {<__gnu_cxx::new_allocator<vbucket_state*>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<vbucket_state*, std::allocator<vbucket_state*> >::_Vector_impl_data> = {_M_start = 0x7fd8c3811400, _M_finish = 0x7fd8c3811800,
                      _M_end_of_storage = 0x7fd8c3811800}, <No data fields>}}, <No data fields>}
              i = 0
              numKvs = 8
      #6  0x000000000074680c in Warmup::initialize() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:1292
              session_stats = {m_type = nlohmann::detail::object, m_value = {object = 0x7fd9b4167d20, array = 0x7fd9b4167d20, string = 0x7fd9b4167d20,
                  binary = 0x7fd9b4167d20, boolean = 32, number_integer = 140573006003488, number_unsigned = 140573006003488,
                  number_float = 6.9452292998958726e-310}}
      ---Type <return> to continue, or q <return> to quit---
              it = {m_object = <optimized out>, m_it = {object_iterator = {_M_node = <optimized out>}, array_iterator = {_M_current = 0x0},
                  primitive_iterator = {static begin_value = 0, static end_value = 1, m_it = <optimized out>}}}
      #7  0x000000000075111d in WarmupInitialize::run (this=0x7fd8ccb15040)
          at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:163
              phosphor_internal_category_enabled_162 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}
              phosphor_internal_category_enabled_temp_162 = <optimized out>
              phosphor_internal_tpi_162 = {category = 0x0, name = 0x0, type = phosphor::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}},
                argument_types = {_M_elems = {phosphor::is_bool, phosphor::is_bool}}}
              phosphor_internal_guard_162 = {tpi = 0x105ea80 <WarmupInitialize::run()::phosphor_internal_tpi_162>, enabled = true,
                arg1 = {<No data fields>}, arg2 = {<No data fields>}, start = {__d = {__r = 6404794305504528}}}
      #8  0x0000000000aa22f2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68
              guard = {previous = 0x0}
      #9  0x0000000000a9f505 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fd9967ea840)
          at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189
              scheduleOverhead = <optimized out>
              start = {__d = {__r = 6404794305502636}}
              runAgain = false
              proxy = @0x7fd9ac40d580: <error reading variable>
      #10 0x0000000000bf4690 in operator() (this=0x7fd9967ea840)
          at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
              fn = @0x7fd9967ea840: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7fd9ac40d580, tiny = {
                    __data = "\200\325@\254\331\177\000\000\000\000\000\000\000\000\000\000\220{;\272\331\177\000\000 ,\001\000\000\000\000\000X\000\000\000\000\000\000\000\020\251~\226\331\177\000", __align = {<No data fields>}}},
                call_ = 0xa9fbc0 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,
                exec_ = 0xa9e310 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}
      #11 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (
          this=this@entry=0x7fd9b9f2a800, thread=...,
          task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1965.x86_64.debug, CU 0xa1d7cd1, DIE 0xa25bc06>)
          at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
              rctx = {
                prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}
              startTime = {__d = {__r = 6404794305497956}}
              stats = {expired = false, waitTime = {__r = 22556}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 6404794305475400}}, requestId = 0}
      #12 0x0000000000bdc44a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fd9b9f2a800, thread=...)
      

      QE-TEST:

      git fetch "https://review.couchbase.org/TAF" refs/changes/88/166488/1 && git checkout FETCH_HEAD
      guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/ankush_temp_job.ini -p bucket_storage=couchstore,rerun=false,bucket_eviction_policy=fullEviction,init_loading=False -t storage.magma.magma_rebalance.MagmaRebalance.test_data_load_collections_with_swap_rebalance,num_items=500000,doc_size=256,nodes_init=2,nodes_swap=2,standard_buckets=1,magma_buckets=0,bucket_storage=couchstore,data_load_stage=during,sdk_timeout=60,vbuckets=1024,key_size=12,replicas=1,infra_log_level=debug,log_level=debug,skip_cleaup=True,randomize_value=True,bucket_eviction_policy=fullEviction,infra_log_level=debug,log_level=debug,init_loading=False,fragmentation=50,skip_cleanup=True,autoCompactionDefined=true,iterations=1,enable_dp=True,num_collections=101,num_scopes=1,bucket_ram_quota=2056,skip_cleanup=True,sdk_client_pool=False,ops_rate=15000,doc_ops=create,create_perc=50,delete_perc=50,update_perc=0,num_collections_to_drop=0 -m rest'
      

      Attachments

        No reviews matched the request. Check your Options in the drop-down menu of this sections header.

        Activity

          People

            ankush.sharma Ankush Sharma
            ankush.sharma Ankush Sharma
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Gerrit Reviews

                There are no open Gerrit changes

                PagerDuty