Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-4889

ep-engine segmentation fault in shutdown path

    XMLWordPrintable

Details

    • Bug
    • Resolution: Fixed
    • Critical
    • 1.8.1
    • 1.8.1
    • couchbase-bucket
    • Security Level: Public
    • None

    Description

      I was exercising new bucket engine fixes and got core that looks like it was ep-engine's problem of destroying some mutex before we stopped using it.

      [Thread debugging using libthread_db enabled]
      Using host libthread_db library "/lib/i386-linux-gnu/i686/cmov/libthread_db.so.1".
      Core was generated by `/root/src/altoros/moxi/repo18/install/bin/memcached.orig -u root -X /root/src/a'.
      Program terminated with signal 6, Aborted.
      #0 0xf7fe0430 in __kernel_vsyscall ()
      (gdb) bt
      #0 0xf7fe0430 in __kernel_vsyscall ()
      #1 0xf7df1941 in *__GI_raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
      #2 0xf7df4d72 in *__GI_abort () at abort.c:92
      #3 0xf429947a in Mutex::acquire (this=0xf23976f0) at mutex.cc:62
      #4 0xf4279b92 in lock (this=0xe76fe000) at locks.hh:48
      #5 LockHolder (m=..., this=0xe76fe000) at locks.hh:26
      #6 notify (this=0xf23976f0) at tapconnmap.hh:192
      #7 EventuallyPersistentEngine::notifyNotificationThread (this=0xf1a0ad88) at ep_engine.cc:3273
      #8 0xf4244d00 in CheckpointVisitor::visitBucket (this=0xe567e6e8, vb=...) at checkpoint_remover.cc:28
      #9 0xf42504c1 in VBCBAdaptor::callback (this=0xebea8e70, d=..., t=...) at ep.cc:2909
      #10 0xf424eafd in Task::run (this=0xeec6c3f8, d=..., t=...) at dispatcher.hh:139
      #11 0xf424b8ce in Dispatcher::run (this=0xeaaf5140) at dispatcher.cc:123
      #12 0xf424de05 in launch_dispatcher_thread (arg=0xeaaf5140) at dispatcher.cc:28
      #13 0xf7f29c39 in start_thread (arg=0xe76feb70) at pthread_create.c:304
      #14 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
      (gdb) thread app al bt

      Thread 12 (Thread 0xdfcffb70 (LWP 23270)):
      #0 0xf7fe0430 in __kernel_vsyscall ()
      #1 0xf7f2ae65 in pthread_join (threadid=3901750128, thread_return=0x0) at pthread_join.c:89
      #2 0xf424b012 in Dispatcher::stop (this=0xe995aa60, force=true) at dispatcher.cc:162
      #3 0xf425bab6 in EventuallyPersistentStore::~EventuallyPersistentStore (this=0xf2783ce0, __in_chrg=<optimized out>) at ep.cc:686
      #4 0xf428a34b in EventuallyPersistentEngine::~EventuallyPersistentEngine (this=0xf1a0ad88, __in_chrg=<optimized out>) at ep_engine.h:502
      #5 0xf427d524 in EvpDestroy (handle=0xf1a0ad88, force=true) at ep_engine.cc:89
      #6 0xf7fd121a in engine_shutdown_thread (arg=0xf1a0aaf0) at bucket_engine.c:1455
      #7 0xf7f29c39 in start_thread (arg=0xdfcffb70) at pthread_create.c:304
      #8 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

      Thread 11 (Thread 0xf558ab70 (LWP 22324)):
      #0 0xf7fe0430 in __kernel_vsyscall ()
      #1 0xf7e97b96 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
      #2 0xf7f94a6c in ?? () from /usr/lib/libevent-2.0.so.5
      #3 0xf7f7ed27 in event_base_loop () from /usr/lib/libevent-2.0.so.5
      #4 0x0805b957 in worker_libevent (arg=0xab58fd8) at daemon/thread.c:304
      #5 0xf7f29c39 in start_thread (arg=0xf558ab70) at pthread_create.c:304
      #6 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

      Thread 10 (Thread 0xf7dc56c0 (LWP 22317)):
      #0 0xf7fe0430 in __kernel_vsyscall ()
      #1 0xf7e97b96 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
      #2 0xf7f94a6c in ?? () from /usr/lib/libevent-2.0.so.5
      #3 0xf7f7ed27 in event_base_loop () from /usr/lib/libevent-2.0.so.5
      #4 0x0805a954 in main (argc=17, argv=0xffffd1b4) at daemon/memcached.c:7772

      Thread 9 (Thread 0xf4d89b70 (LWP 22325)):
      #0 0xf7fe0430 in __kernel_vsyscall ()
      #1 0xf7e97b96 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
      #2 0xf7f94a6c in ?? () from /usr/lib/libevent-2.0.so.5
      #3 0xf7f7ed27 in event_base_loop () from /usr/lib/libevent-2.0.so.5
      #4 0x0805b957 in worker_libevent (arg=0xab59068) at daemon/thread.c:304
      #5 0xf7f29c39 in start_thread (arg=0xf4d89b70) at pthread_create.c:304
      #6 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

      Thread 8 (Thread 0xe6efdb70 (LWP 23139)):
      #0 0xf7fe0430 in __kernel_vsyscall ()
      #1 0xf7f2e20a in __pthread_cond_wait (cond=0xefbb5e74, mutex=0xefbb5e54) at pthread_cond_wait.c:153
      #2 0xf424be64 in wait (this=0xefbb5e50) at syncobject.hh:31
      #3 Dispatcher::run (this=0xefbb5e48) at dispatcher.cc:89
      #4 0xf424de05 in launch_dispatcher_thread (arg=0xefbb5e48) at dispatcher.cc:28
      #5 0xf7f29c39 in start_thread (arg=0xe6efdb70) at pthread_create.c:304
      #6 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

      Thread 7 (Thread 0xf6d8db70 (LWP 22321)):
      #0 0xf7fe0430 in __kernel_vsyscall ()
      #1 0xf7e97b96 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
      #2 0xf7f94a6c in ?? () from /usr/lib/libevent-2.0.so.5
      #3 0xf7f7ed27 in event_base_loop () from /usr/lib/libevent-2.0.so.5
      #4 0x0805b957 in worker_libevent (arg=0xab58e28) at daemon/thread.c:304
      #5 0xf7f29c39 in start_thread (arg=0xf6d8db70) at pthread_create.c:304
      #6 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

      Thread 6 (Thread 0xf7dc4b70 (LWP 22319)):
      #0 0xf7fe0430 in __kernel_vsyscall ()
      #1 0xf7e87e3b in read () at ../sysdeps/unix/syscall-template.S:82
      #2 0xf7e310cb in _IO_new_file_underflow (fp=0xf7f20440) at fileops.c:606
      #3 0xf7e3296b in _IO_default_uflow (fp=0xf7f20440) at genops.c:440
      #4 0xf7e33d98 in *_GI__uflow (fp=0xf7f20440) at genops.c:394
      --Type <return> to continue, or q <return> to quit--
      #5 0xf7e2967c in _IO_getc (fp=0xf7f20440) at getc.c:41
      #6 0xf7fd870c in check_stdin_thread (arg=0x804aa70) at extensions/daemon/stdin_check.c:19
      #7 0xf7f29c39 in start_thread (arg=0xf7dc4b70) at pthread_create.c:304
      #8 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

      Thread 5 (Thread 0xf758eb70 (LWP 22320)):
      #0 0xf7fe0430 in __kernel_vsyscall ()
      #1 0xf7e61bc6 in nanosleep () at ../sysdeps/unix/syscall-template.S:82
      #2 0xf7e619f0 in __sleep (seconds=0) at ../sysdeps/unix/sysv/linux/sleep.c:138
      #3 0x0805deab in check_isasl_db_thread (arg=0x806a4c0) at daemon/isasl.c:233
      #4 0xf7f29c39 in start_thread (arg=0xf758eb70) at pthread_create.c:304
      #5 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

      Thread 4 (Thread 0xf5d8bb70 (LWP 22323)):
      #0 0xf7fe0430 in __kernel_vsyscall ()
      #1 0xf7e97b96 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
      #2 0xf7f94a6c in ?? () from /usr/lib/libevent-2.0.so.5
      #3 0xf7f7ed27 in event_base_loop () from /usr/lib/libevent-2.0.so.5
      #4 0x0805b957 in worker_libevent (arg=0xab58f48) at daemon/thread.c:304
      #5 0xf7f29c39 in start_thread (arg=0xf5d8bb70) at pthread_create.c:304
      #6 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

      Thread 3 (Thread 0xe88ffb70 (LWP 23140)):
      #0 0xf7fe0430 in __kernel_vsyscall ()
      #1 0xf7f2e703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
      #2 0xf4249139 in wait (tv=..., this=0xe995aa68) at syncobject.hh:42
      #3 IdleTask::run (this=0xe995b7c8, d=...) at dispatcher.cc:286
      #4 0xf424b8ce in Dispatcher::run (this=0xe995aa60) at dispatcher.cc:123
      #5 0xf424de05 in launch_dispatcher_thread (arg=0xe995aa60) at dispatcher.cc:28
      #6 0xf7f29c39 in start_thread (arg=0xe88ffb70) at pthread_create.c:304
      #7 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

      Thread 2 (Thread 0xf658cb70 (LWP 22322)):
      #0 0xf7fe0430 in __kernel_vsyscall ()
      #1 0xf7e97b96 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
      #2 0xf7f94a6c in ?? () from /usr/lib/libevent-2.0.so.5
      #3 0xf7f7ed27 in event_base_loop () from /usr/lib/libevent-2.0.so.5
      #4 0x0805b957 in worker_libevent (arg=0xab58eb8) at daemon/thread.c:304
      #5 0xf7f29c39 in start_thread (arg=0xf658cb70) at pthread_create.c:304
      #6 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

      Thread 1 (Thread 0xe76feb70 (LWP 23141)):
      #0 0xf7fe0430 in __kernel_vsyscall ()
      #1 0xf7df1941 in *__GI_raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
      #2 0xf7df4d72 in *__GI_abort () at abort.c:92
      #3 0xf429947a in Mutex::acquire (this=0xf23976f0) at mutex.cc:62
      #4 0xf4279b92 in lock (this=0xe76fe000) at locks.hh:48
      #5 LockHolder (m=..., this=0xe76fe000) at locks.hh:26
      #6 notify (this=0xf23976f0) at tapconnmap.hh:192
      #7 EventuallyPersistentEngine::notifyNotificationThread (this=0xf1a0ad88) at ep_engine.cc:3273
      #8 0xf4244d00 in CheckpointVisitor::visitBucket (this=0xe567e6e8, vb=...) at checkpoint_remover.cc:28
      #9 0xf42504c1 in VBCBAdaptor::callback (this=0xebea8e70, d=..., t=...) at ep.cc:2909
      #10 0xf424eafd in Task::run (this=0xeec6c3f8, d=..., t=...) at dispatcher.hh:139
      #11 0xf424b8ce in Dispatcher::run (this=0xeaaf5140) at dispatcher.cc:123
      #12 0xf424de05 in launch_dispatcher_thread (arg=0xeaaf5140) at dispatcher.cc:28
      #13 0xf7f29c39 in start_thread (arg=0xe76feb70) at pthread_create.c:304
      #14 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

      Attachments

        No reviews matched the request. Check your Options in the drop-down menu of this sections header.

        Activity

          People

            mikew Mike Wiederhold [X] (Inactive)
            alkondratenko Aleksey Kondratenko (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            1 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Gerrit Reviews

                There are no open Gerrit changes

                PagerDuty