Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-3838

memcached hangs while recreating a bucket with same name if ep-engine deletion is busy waiting for the flusher

    XMLWordPrintable

Details

    • Bug
    • Resolution: Fixed
    • Major
    • 2.0-beta
    • 1.7 alpha 2
    • None
    • Security Level: Public
    • None
    • basestar-257

    Description

      Thread 13 (Thread 0x7fe468fcc710 (LWP 22290)):
      #0 0x00007fe4692a747d in read () from /lib/libc.so.6
      #1 0x00007fe469242348 in _IO_file_underflow () from /lib/libc.so.6
      #2 0x00007fe469243eee in _IO_default_uflow () from /lib/libc.so.6
      #3 0x00007fe46923ac7b in getc () from /lib/libc.so.6
      #4 0x00007fe468fcd965 in check_stdin_thread (arg=0x403b70) at extensions/daemon/stdin_check.c:19
      #5 0x00007fe4695579ca in start_thread () from /lib/libpthread.so.0
      #6 0x00007fe4692b56cd in clone () from /lib/libc.so.6
      #7 0x0000000000000000 in ?? ()

      Thread 12 (Thread 0x7fe4687cb710 (LWP 22291)):
      #0 0x00007fe46927935d in nanosleep () from /lib/libc.so.6
      #1 0x00007fe4692791d0 in sleep () from /lib/libc.so.6
      #2 0x0000000000416618 in check_isasl_db_thread (arg=<value optimized out>) at daemon/isasl.c:233
      #3 0x00007fe4695579ca in start_thread () from /lib/libpthread.so.0
      #4 0x00007fe4692b56cd in clone () from /lib/libc.so.6
      #5 0x0000000000000000 in ?? ()

      Thread 11 (Thread 0x7fe467dbf710 (LWP 22292)):
      #0 0x00007fe4692b5cc3 in epoll_wait () from /lib/libc.so.6
      #1 0x00007fe469e2095c in epoll_dispatch (base=0x1b1e0b0, tv=<value optimized out>) at epoll.c:404
      #2 0x00007fe469e0f2cc in event_base_loop (base=0x1b1e0b0, flags=<value optimized out>) at event.c:1558
      #3 0x0000000000414824 in worker_libevent (arg=0x1b1db50) at daemon/thread.c:304
      #4 0x00007fe4695579ca in start_thread () from /lib/libpthread.so.0
      #5 0x00007fe4692b56cd in clone () from /lib/libc.so.6
      #6 0x0000000000000000 in ?? ()

      Thread 10 (Thread 0x7fe4675be710 (LWP 22293)):
      #0 0x00007fe4692b5cc3 in epoll_wait () from /lib/libc.so.6
      #1 0x00007fe469e2095c in epoll_dispatch (base=0x1b1e950, tv=<value optimized out>) at epoll.c:404
      #2 0x00007fe469e0f2cc in event_base_loop (base=0x1b1e950, flags=<value optimized out>) at event.c:1558
      #3 0x0000000000414824 in worker_libevent (arg=0x1b1dc50) at daemon/thread.c:304
      #4 0x00007fe4695579ca in start_thread () from /lib/libpthread.so.0
      #5 0x00007fe4692b56cd in clone () from /lib/libc.so.6
      #6 0x0000000000000000 in ?? ()

      Thread 9 (Thread 0x7fe466dbd710 (LWP 22294)):
      #0 0x00007fe4692b5cc3 in epoll_wait () from /lib/libc.so.6
      #1 0x00007fe469e2095c in epoll_dispatch (base=0x1b1f1f0, tv=<value optimized out>) at epoll.c:404
      #2 0x00007fe469e0f2cc in event_base_loop (base=0x1b1f1f0, flags=<value optimized out>) at event.c:1558
      #3 0x0000000000414824 in worker_libevent (arg=0x1b1dd50) at daemon/thread.c:304
      #4 0x00007fe4695579ca in start_thread () from /lib/libpthread.so.0
      #5 0x00007fe4692b56cd in clone () from /lib/libc.so.6
      #6 0x0000000000000000 in ?? ()

      Thread 8 (Thread 0x7fe4665bc710 (LWP 22295)):
      #0 0x00007fe4692b5cc3 in epoll_wait () from /lib/libc.so.6
      #1 0x00007fe469e2095c in epoll_dispatch (base=0x1b1fa90, tv=<value optimized out>) at epoll.c:404
      #2 0x00007fe469e0f2cc in event_base_loop (base=0x1b1fa90, flags=<value optimized out>) at event.c:1558
      #3 0x0000000000414824 in worker_libevent (arg=0x1b1de50) at daemon/thread.c:304
      #4 0x00007fe4695579ca in start_thread () from /lib/libpthread.so.0
      #5 0x00007fe4692b56cd in clone () from /lib/libc.so.6
      #6 0x0000000000000000 in ?? ()
      --Type <return> to continue, or q <return> to quit--

      Thread 7 (Thread 0x7fe465dbb710 (LWP 22296)):
      #0 0x00007fe4692b5cc3 in epoll_wait () from /lib/libc.so.6
      #1 0x00007fe469e2095c in epoll_dispatch (base=0x1b20430, tv=<value optimized out>) at epoll.c:404
      #2 0x00007fe469e0f2cc in event_base_loop (base=0x1b20430, flags=<value optimized out>) at event.c:1558
      #3 0x0000000000414824 in worker_libevent (arg=0x1b1df50) at daemon/thread.c:304
      #4 0x00007fe4695579ca in start_thread () from /lib/libpthread.so.0
      #5 0x00007fe4692b56cd in clone () from /lib/libc.so.6
      #6 0x0000000000000000 in ?? ()

      Thread 6 (Thread 0x7fe462d21710 (LWP 22380)):
      #0 0x00007fe46955cbc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
      #1 0x00007fe467dc370a in engine_shutdown_thread (arg=0x1b9a700) at bucket_engine.c:1137
      #2 0x00007fe4695579ca in start_thread () from /lib/libpthread.so.0
      #3 0x00007fe4692b56cd in clone () from /lib/libc.so.6
      #4 0x0000000000000000 in ?? ()

      Thread 5 (Thread 0x7fe464524710 (LWP 22440)):
      #0 0x00007fe46955c85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
      #1 0x00007fe465275ded in SyncObject::wait (this=0x7fe45c9688e0) at syncobject.hh:31
      #2 Dispatcher::run (this=0x7fe45c9688e0) at dispatcher.cc:85
      #3 0x00007fe465276793 in launch_dispatcher_thread (arg=0x7fe45c96892c) at dispatcher.cc:28
      #4 0x00007fe4695579ca in start_thread () from /lib/libpthread.so.0
      #5 0x00007fe4692b56cd in clone () from /lib/libc.so.6
      #6 0x0000000000000000 in ?? ()

      Thread 4 (Thread 0x7fe463d23710 (LWP 22441)):
      #0 __atomic_add (this=0x7fe44c78ba10, cursor=..., barrier=0, items=...) at /usr/include/c++/4.3/ext/atomicity.h:55
      #1 __atomic_add_dispatch (this=0x7fe44c78ba10, cursor=..., barrier=0, items=...) at /usr/include/c++/4.3/ext/atomicity.h:98
      #2 std::tr1::Sp_counted_base<(_gnu_cxx::_Lock_policy)2>::_M_add_ref_copy (this=0x7fe44c78ba10, cursor=..., barrier=0, items=...)
      at /usr/include/c++/4.3/tr1_impl/boost_sp_counted_base.h:135
      #3 __shared_count (this=0x7fe44c78ba10, cursor=..., barrier=0, items=...) at /usr/include/c++/4.3/tr1/boost_sp_shared_count.h:162
      #4 __shared_ptr (this=0x7fe44c78ba10, cursor=..., barrier=0, items=...) at /usr/include/c++/4.3/tr1_impl/boost_shared_ptr.h:206
      #5 shared_ptr (this=0x7fe44c78ba10, cursor=..., barrier=0, items=...) at /usr/include/c++/4.3/tr1_impl/boost_shared_ptr.h:822
      #6 __gnu_cxx::new_allocator<std::tr1::shared_ptr<QueuedItem> >::construct (this=0x7fe44c78ba10, cursor=..., barrier=0, items=...)
      at /usr/include/c++/4.3/ext/new_allocator.h:108
      #7 std::vector<std::tr1::shared_ptr<QueuedItem>, std::allocator<std::tr1::shared_ptr<QueuedItem> > >::push_back (this=0x7fe44c78ba10,
      cursor=..., barrier=0, items=...) at /usr/include/c++/4.3/bits/stl_vector.h:690
      #8 CheckpointManager::getAllItemsFromCurrentPosition (this=0x7fe44c78ba10, cursor=..., barrier=0, items=...) at checkpoint.cc:521
      #9 0x00007fe46526c409 in CheckpointManager::getAllItemsForPersistence (this=0x7fe427e0bf80, items=...) at checkpoint.cc:553
      #10 0x00007fe46527d1d5 in EventuallyPersistentStore::beginFlush (this=0x7fe45c417210) at ep.cc:1383
      #11 0x00007fe4652b2d4d in Flusher::doFlush (this=0x7fe45e106f10) at flusher.cc:228
      #12 0x00007fe4652b3715 in Flusher::step (this=0x7fe44c78ba10, d=..., tid=...) at flusher.cc:154
      #13 0x00007fe4652b3c4a in FlusherStepper::callback (this=<value optimized out>, d=..., t=<value optimized out>) at flusher.cc:23
      #14 0x00007fe465276e9a in Task::run (this=<value optimized out>, d=..., t=<value optimized out>) at dispatcher.hh:139
      #15 0x00007fe465275b82 in Dispatcher::run (this=0x7fe45c55f100) at dispatcher.cc:119
      #16 0x00007fe465276793 in launch_dispatcher_thread (arg=0x7fe44c78ba10) at dispatcher.cc:28
      #17 0x00007fe4695579ca in start_thread () from /lib/libpthread.so.0
      #18 0x00007fe4692b56cd in clone () from /lib/libc.so.6
      #19 0x0000000000000000 in ?? ()

      Thread 3 (Thread 0x7fe464d25710 (LWP 22442)):
      #0 0x00007fe46955f464 in __lll_lock_wait () from /lib/libpthread.so.0
      #1 0x00007fe46955a5d9 in _L_lock_953 () from /lib/libpthread.so.0
      --Type <return> to continue, or q <return> to quit--
      #2 0x00007fe46955a3fb in pthread_mutex_lock () from /lib/libpthread.so.0
      #3 0x00007fe46526f64a in Mutex::acquire (this=0x7fe464d24ad0, m=...) at mutex.hh:64
      #4 LockHolder::lock (this=0x7fe464d24ad0, m=...) at locks.hh:48
      #5 LockHolder (this=0x7fe464d24ad0, m=...) at locks.hh:26
      #6 0x00007fe46526e182 in CheckpointManager::removeClosedUnrefCheckpoints (this=0x7fe427e0bf80, vbucket=..., items=...,
      newOpenCheckpointCreated=@0xffffffffffffffff) at checkpoint.cc:403
      #7 0x00007fe465273b36 in CheckpointVisitor::visitBucket (this=0x7fe45407f0a0, vb=...) at checkpoint_remover.cc:27
      #8 0x00007fe465283f40 in VBCBAdaptor::callback (this=0x7fe44c855660) at ep.cc:2140
      #9 0x00007fe465276e9a in Task::run (this=<value optimized out>, d=..., t=<value optimized out>) at dispatcher.hh:139
      #10 0x00007fe465275b82 in Dispatcher::run (this=0x7fe45e105d40) at dispatcher.cc:119
      #11 0x00007fe465276793 in launch_dispatcher_thread (arg=0x7fe427e0bf90) at dispatcher.cc:28
      #12 0x00007fe4695579ca in start_thread () from /lib/libpthread.so.0
      #13 0x00007fe4692b56cd in clone () from /lib/libc.so.6
      #14 0x0000000000000000 in ?? ()

      Thread 2 (Thread 0x7fe441ffb710 (LWP 23109)):
      #0 0x00007fe46955903d in pthread_join () from /lib/libpthread.so.0
      #1 0x00007fe465273e78 in Dispatcher::stop (this=0x7fe45c55f100, force=true) at dispatcher.cc:158
      #2 0x00007fe465279d66 in ~EventuallyPersistentStore (this=0x7fe45c417210, __in_chrg=<value optimized out>) at ep.cc:415

      #3 0x00007fe46529755f in ~EventuallyPersistentEngine (handle=0x7fe45d62bd70, force=<value optimized out>) at ep_engine.h:499
      #4 EvpDestroy (handle=0x7fe45d62bd70, force=<value optimized out>) at ep_engine.cc:97
      #5 0x00007fe467dc355d in engine_shutdown_thread (arg=0x7fe45d2eb010) at bucket_engine.c:1099
      #6 0x00007fe4695579ca in start_thread () from /lib/libpthread.so.0
      #7 0x00007fe4692b56cd in clone () from /lib/libc.so.6
      #8 0x0000000000000000 in ?? ()

      Thread 1 (Thread 0x7fe46a45d700 (LWP 22284)):
      #0 0x00007fe4692b5cc3 in epoll_wait () from /lib/libc.so.6
      #1 0x00007fe469e2095c in epoll_dispatch (base=0x1b12a90, tv=<value optimized out>) at epoll.c:404
      #2 0x00007fe469e0f2cc in event_base_loop (base=0x1b12a90, flags=<value optimized out>) at event.c:1558
      #3 0x000000000040ae28 in main (argc=<value optimized out>, argv=0x7ffff052fff8) at daemon/memcached.c:7459
      (gdb)
      (gdb) quit
      A debugging session is active.

      Inferior 1 [process 22284] will be detached.

      Attachments

        No reviews matched the request. Check your Options in the drop-down menu of this sections header.

        Activity

          People

            chiyoung Chiyoung Seo (Inactive)
            farshid Farshid Ghods (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            0 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Gerrit Reviews

                There are no open Gerrit changes

                PagerDuty