Details
-
Bug
-
Resolution: Fixed
-
Critical
-
1.8.1
-
Security Level: Public
-
None
Description
I was exercising new bucket engine fixes and got core that looks like it was ep-engine's problem of destroying some mutex before we stopped using it.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/i386-linux-gnu/i686/cmov/libthread_db.so.1".
Core was generated by `/root/src/altoros/moxi/repo18/install/bin/memcached.orig -u root -X /root/src/a'.
Program terminated with signal 6, Aborted.
#0 0xf7fe0430 in __kernel_vsyscall ()
(gdb) bt
#0 0xf7fe0430 in __kernel_vsyscall ()
#1 0xf7df1941 in *__GI_raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
#2 0xf7df4d72 in *__GI_abort () at abort.c:92
#3 0xf429947a in Mutex::acquire (this=0xf23976f0) at mutex.cc:62
#4 0xf4279b92 in lock (this=0xe76fe000) at locks.hh:48
#5 LockHolder (m=..., this=0xe76fe000) at locks.hh:26
#6 notify (this=0xf23976f0) at tapconnmap.hh:192
#7 EventuallyPersistentEngine::notifyNotificationThread (this=0xf1a0ad88) at ep_engine.cc:3273
#8 0xf4244d00 in CheckpointVisitor::visitBucket (this=0xe567e6e8, vb=...) at checkpoint_remover.cc:28
#9 0xf42504c1 in VBCBAdaptor::callback (this=0xebea8e70, d=..., t=...) at ep.cc:2909
#10 0xf424eafd in Task::run (this=0xeec6c3f8, d=..., t=...) at dispatcher.hh:139
#11 0xf424b8ce in Dispatcher::run (this=0xeaaf5140) at dispatcher.cc:123
#12 0xf424de05 in launch_dispatcher_thread (arg=0xeaaf5140) at dispatcher.cc:28
#13 0xf7f29c39 in start_thread (arg=0xe76feb70) at pthread_create.c:304
#14 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
(gdb) thread app al bt
Thread 12 (Thread 0xdfcffb70 (LWP 23270)):
#0 0xf7fe0430 in __kernel_vsyscall ()
#1 0xf7f2ae65 in pthread_join (threadid=3901750128, thread_return=0x0) at pthread_join.c:89
#2 0xf424b012 in Dispatcher::stop (this=0xe995aa60, force=true) at dispatcher.cc:162
#3 0xf425bab6 in EventuallyPersistentStore::~EventuallyPersistentStore (this=0xf2783ce0, __in_chrg=<optimized out>) at ep.cc:686
#4 0xf428a34b in EventuallyPersistentEngine::~EventuallyPersistentEngine (this=0xf1a0ad88, __in_chrg=<optimized out>) at ep_engine.h:502
#5 0xf427d524 in EvpDestroy (handle=0xf1a0ad88, force=true) at ep_engine.cc:89
#6 0xf7fd121a in engine_shutdown_thread (arg=0xf1a0aaf0) at bucket_engine.c:1455
#7 0xf7f29c39 in start_thread (arg=0xdfcffb70) at pthread_create.c:304
#8 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
Thread 11 (Thread 0xf558ab70 (LWP 22324)):
#0 0xf7fe0430 in __kernel_vsyscall ()
#1 0xf7e97b96 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
#2 0xf7f94a6c in ?? () from /usr/lib/libevent-2.0.so.5
#3 0xf7f7ed27 in event_base_loop () from /usr/lib/libevent-2.0.so.5
#4 0x0805b957 in worker_libevent (arg=0xab58fd8) at daemon/thread.c:304
#5 0xf7f29c39 in start_thread (arg=0xf558ab70) at pthread_create.c:304
#6 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
Thread 10 (Thread 0xf7dc56c0 (LWP 22317)):
#0 0xf7fe0430 in __kernel_vsyscall ()
#1 0xf7e97b96 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
#2 0xf7f94a6c in ?? () from /usr/lib/libevent-2.0.so.5
#3 0xf7f7ed27 in event_base_loop () from /usr/lib/libevent-2.0.so.5
#4 0x0805a954 in main (argc=17, argv=0xffffd1b4) at daemon/memcached.c:7772
Thread 9 (Thread 0xf4d89b70 (LWP 22325)):
#0 0xf7fe0430 in __kernel_vsyscall ()
#1 0xf7e97b96 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
#2 0xf7f94a6c in ?? () from /usr/lib/libevent-2.0.so.5
#3 0xf7f7ed27 in event_base_loop () from /usr/lib/libevent-2.0.so.5
#4 0x0805b957 in worker_libevent (arg=0xab59068) at daemon/thread.c:304
#5 0xf7f29c39 in start_thread (arg=0xf4d89b70) at pthread_create.c:304
#6 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
Thread 8 (Thread 0xe6efdb70 (LWP 23139)):
#0 0xf7fe0430 in __kernel_vsyscall ()
#1 0xf7f2e20a in __pthread_cond_wait (cond=0xefbb5e74, mutex=0xefbb5e54) at pthread_cond_wait.c:153
#2 0xf424be64 in wait (this=0xefbb5e50) at syncobject.hh:31
#3 Dispatcher::run (this=0xefbb5e48) at dispatcher.cc:89
#4 0xf424de05 in launch_dispatcher_thread (arg=0xefbb5e48) at dispatcher.cc:28
#5 0xf7f29c39 in start_thread (arg=0xe6efdb70) at pthread_create.c:304
#6 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
Thread 7 (Thread 0xf6d8db70 (LWP 22321)):
#0 0xf7fe0430 in __kernel_vsyscall ()
#1 0xf7e97b96 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
#2 0xf7f94a6c in ?? () from /usr/lib/libevent-2.0.so.5
#3 0xf7f7ed27 in event_base_loop () from /usr/lib/libevent-2.0.so.5
#4 0x0805b957 in worker_libevent (arg=0xab58e28) at daemon/thread.c:304
#5 0xf7f29c39 in start_thread (arg=0xf6d8db70) at pthread_create.c:304
#6 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
Thread 6 (Thread 0xf7dc4b70 (LWP 22319)):
#0 0xf7fe0430 in __kernel_vsyscall ()
#1 0xf7e87e3b in read () at ../sysdeps/unix/syscall-template.S:82
#2 0xf7e310cb in _IO_new_file_underflow (fp=0xf7f20440) at fileops.c:606
#3 0xf7e3296b in _IO_default_uflow (fp=0xf7f20440) at genops.c:440
#4 0xf7e33d98 in *_GI__uflow (fp=0xf7f20440) at genops.c:394
--Type <return> to continue, or q <return> to quit--
#5 0xf7e2967c in _IO_getc (fp=0xf7f20440) at getc.c:41
#6 0xf7fd870c in check_stdin_thread (arg=0x804aa70) at extensions/daemon/stdin_check.c:19
#7 0xf7f29c39 in start_thread (arg=0xf7dc4b70) at pthread_create.c:304
#8 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
Thread 5 (Thread 0xf758eb70 (LWP 22320)):
#0 0xf7fe0430 in __kernel_vsyscall ()
#1 0xf7e61bc6 in nanosleep () at ../sysdeps/unix/syscall-template.S:82
#2 0xf7e619f0 in __sleep (seconds=0) at ../sysdeps/unix/sysv/linux/sleep.c:138
#3 0x0805deab in check_isasl_db_thread (arg=0x806a4c0) at daemon/isasl.c:233
#4 0xf7f29c39 in start_thread (arg=0xf758eb70) at pthread_create.c:304
#5 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
Thread 4 (Thread 0xf5d8bb70 (LWP 22323)):
#0 0xf7fe0430 in __kernel_vsyscall ()
#1 0xf7e97b96 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
#2 0xf7f94a6c in ?? () from /usr/lib/libevent-2.0.so.5
#3 0xf7f7ed27 in event_base_loop () from /usr/lib/libevent-2.0.so.5
#4 0x0805b957 in worker_libevent (arg=0xab58f48) at daemon/thread.c:304
#5 0xf7f29c39 in start_thread (arg=0xf5d8bb70) at pthread_create.c:304
#6 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
Thread 3 (Thread 0xe88ffb70 (LWP 23140)):
#0 0xf7fe0430 in __kernel_vsyscall ()
#1 0xf7f2e703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf4249139 in wait (tv=..., this=0xe995aa68) at syncobject.hh:42
#3 IdleTask::run (this=0xe995b7c8, d=...) at dispatcher.cc:286
#4 0xf424b8ce in Dispatcher::run (this=0xe995aa60) at dispatcher.cc:123
#5 0xf424de05 in launch_dispatcher_thread (arg=0xe995aa60) at dispatcher.cc:28
#6 0xf7f29c39 in start_thread (arg=0xe88ffb70) at pthread_create.c:304
#7 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
Thread 2 (Thread 0xf658cb70 (LWP 22322)):
#0 0xf7fe0430 in __kernel_vsyscall ()
#1 0xf7e97b96 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
#2 0xf7f94a6c in ?? () from /usr/lib/libevent-2.0.so.5
#3 0xf7f7ed27 in event_base_loop () from /usr/lib/libevent-2.0.so.5
#4 0x0805b957 in worker_libevent (arg=0xab58eb8) at daemon/thread.c:304
#5 0xf7f29c39 in start_thread (arg=0xf658cb70) at pthread_create.c:304
#6 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
Thread 1 (Thread 0xe76feb70 (LWP 23141)):
#0 0xf7fe0430 in __kernel_vsyscall ()
#1 0xf7df1941 in *__GI_raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
#2 0xf7df4d72 in *__GI_abort () at abort.c:92
#3 0xf429947a in Mutex::acquire (this=0xf23976f0) at mutex.cc:62
#4 0xf4279b92 in lock (this=0xe76fe000) at locks.hh:48
#5 LockHolder (m=..., this=0xe76fe000) at locks.hh:26
#6 notify (this=0xf23976f0) at tapconnmap.hh:192
#7 EventuallyPersistentEngine::notifyNotificationThread (this=0xf1a0ad88) at ep_engine.cc:3273
#8 0xf4244d00 in CheckpointVisitor::visitBucket (this=0xe567e6e8, vb=...) at checkpoint_remover.cc:28
#9 0xf42504c1 in VBCBAdaptor::callback (this=0xebea8e70, d=..., t=...) at ep.cc:2909
#10 0xf424eafd in Task::run (this=0xeec6c3f8, d=..., t=...) at dispatcher.hh:139
#11 0xf424b8ce in Dispatcher::run (this=0xeaaf5140) at dispatcher.cc:123
#12 0xf424de05 in launch_dispatcher_thread (arg=0xeaaf5140) at dispatcher.cc:28
#13 0xf7f29c39 in start_thread (arg=0xe76feb70) at pthread_create.c:304
#14 0xf7e9712e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130