Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-42441

Crash during phosphor shutdown if joined thread didn't explicitly register

    XMLWordPrintable

Details

    Description

      Crash observed during kv-engine-jepsen-post-commit-2652 test: lein trampoline run test --nodes-file ./nodes --username vagrant --ssh-private-key ./resources/vagrantkey --install-path <http://cv.jenkins.couchbase.com/job/kv-engine-jepsen-post-commit/ws/install> --workload=kill --scenario=kill-babysitter --replicas=2 --no-autofailover --durability=0:0:0:100 --eviction-policy=value --cas --use-json-docs --use-subdoc --doc-padding-size=4096 --hashdump --enable-memcached-debug-log-level --enable-tcp-capture --autofailover-timeout=30

      (gdb) bt
      #0  0x00007f0d9916fc19 in std::atomic<phosphor::ChunkLock::State>::compare_exchange_weak (__f=<optimized out>, __s=std::memory_order_seq_cst, __i=phosphor::ChunkLock::State::MasterLocked, __e=<synthetic pointer>: phosphor::ChunkLock::State::Unlocked, this=0x7f0d91470bd8) at /usr/include/c++/7/atomic:291
      #1  std::atomic<phosphor::ChunkLock::State>::compare_exchange_weak (__m=std::memory_order_seq_cst, __i=phosphor::ChunkLock::State::MasterLocked, __e=<synthetic pointer>: phosphor::ChunkLock::State::Unlocked, this=0x7f0d91470bd8) at /usr/include/c++/7/atomic:308
      #2  phosphor::ChunkLock::lockMaster (this=0x7f0d91470bd8) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/phosphor/src/chunk_lock.cc:72
      #3  0x00007f0d9916fc85 in phosphor::MasterChunkLock::lock (this=<optimized out>) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/phosphor/src/chunk_lock.cc:105
      #4  0x00007f0d9917744c in phosphor::TraceLog::evictThreads (this=this@entry=0x7f0d99383600 <phosphor::TraceLog::getInstance()::log_instance>, lh=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/phosphor/src/trace_log.cc:309
      #5  0x00007f0d991774ba in phosphor::TraceLog::stop (this=0x7f0d99383600 <phosphor::TraceLog::getInstance()::log_instance>, lh=..., shutdown=<optimized out>) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/phosphor/src/trace_log.cc:93
      #6  0x00007f0d9917755b in phosphor::TraceLog::stop (this=0x7f0d99383600 <phosphor::TraceLog::getInstance()::log_instance>, shutdown=shutdown@entry=false) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/phosphor/src/trace_log.cc:70
      #7  0x000000000049b617 in deinitializeTracing () at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/tracing.cc:107
      #8  0x000000000044eb9c in memcached_main (argc=<optimized out>, argv=<optimized out>) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/memcached.cc:1566
      #9  0x00007f0d970b7830 in __libc_start_main (main=0x4302d0 <main(int, char**)>, argc=3, argv=0x7fff7e569bb8, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7fff7e569ba8) at ../csu/libc-start.c:291
      #10 0x0000000000446809 in _start ()
      

      (gdb) bt full
      #0  0x00007f0d9916fc19 in std::atomic<phosphor::ChunkLock::State>::compare_exchange_weak (__f=<optimized out>, __s=std::memory_order_seq_cst, __i=phosphor::ChunkLock::State::MasterLocked, __e=<synthetic pointer>: phosphor::ChunkLock::State::Unlocked, this=0x7f0d91470bd8) at /usr/include/c++/7/atomic:291
      No locals.
      #1  std::atomic<phosphor::ChunkLock::State>::compare_exchange_weak (__m=std::memory_order_seq_cst, __i=phosphor::ChunkLock::State::MasterLocked, __e=<synthetic pointer>: phosphor::ChunkLock::State::Unlocked, this=0x7f0d91470bd8) at /usr/include/c++/7/atomic:308
      No locals.
      #2  phosphor::ChunkLock::lockMaster (this=0x7f0d91470bd8) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/phosphor/src/chunk_lock.cc:72
              expected = phosphor::ChunkLock::State::Unlocked
      #3  0x00007f0d9916fc85 in phosphor::MasterChunkLock::lock (this=<optimized out>) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/phosphor/src/chunk_lock.cc:105
      No locals.
      #4  0x00007f0d9917744c in phosphor::TraceLog::evictThreads (this=this@entry=0x7f0d99383600 <phosphor::TraceLog::getInstance()::log_instance>, lh=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/phosphor/src/trace_log.cc:309
              chunk_tenant = 0x7f0d91470bd8
              __for_range = std::unordered_set with 0 elements
              __for_begin = <optimized out>
              __for_end = <optimized out>
      #5  0x00007f0d991774ba in phosphor::TraceLog::stop (this=0x7f0d99383600 <phosphor::TraceLog::getInstance()::log_instance>, lh=..., shutdown=<optimized out>) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/phosphor/src/trace_log.cc:93
              cb = <optimized out>
              shutdown = false
              lh = @0x7fff7e569920: {_M_device = @0x7f0d99383600}
              this = 0x7f0d99383600 <phosphor::TraceLog::getInstance()::log_instance>
              cb = <optimized out>
      #6  0x00007f0d9917755b in phosphor::TraceLog::stop (this=0x7f0d99383600 <phosphor::TraceLog::getInstance()::log_instance>, shutdown=shutdown@entry=false) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/phosphor/src/trace_log.cc:70
              lh = {_M_device = @0x7f0d99383600}
      #7  0x000000000049b617 in deinitializeTracing () at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/tracing.cc:107
              guard = <optimized out>
      #8  0x000000000044eb9c in memcached_main (argc=<optimized out>, argv=<optimized out>) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/memcached.cc:1566
              numa_status = <error: Cannot access memory at address 0x7f0d957a7590>
              parent_monitor = std::unique_ptr<ParentMonitor> = {get() = 0x0}
              prometheusExporter = std::unique_ptr<cb::prometheus::MetricServer> = {get() = 0x0}
              phosphor_internal_category_enabled_1492 = {_M_b = {Segmentation fault (core dumped)
      

      Attachments

        Issue Links

          No reviews matched the request. Check your Options in the drop-down menu of this sections header.

          Activity

            People

              richard.demellow Richard deMellow
              richard.demellow Richard deMellow
              Votes:
              0 Vote for this issue
              Watchers:
              8 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved:

                Gerrit Reviews

                  There are no open Gerrit changes

                  PagerDuty