Details
-
Bug
-
Resolution: Fixed
-
Critical
-
Cheshire-Cat
-
Untriaged
-
-
Unknown
-
Magma-DP-Dec23, Magma: Jan6 - Jan17
Description
1. Create a 1 node cluster, default bucket
2. Rebalance In 3 nodes and load data in parallel
3. Rebalance hung
gdb during rebalance hung:
Thread 32 (Thread 0x7f97e01ff700 (LWP 21183)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97e2bd75a7 in background_thread_sleep (tsdn=<optimized out>,
|
interval=10000000000, info=<optimized out>) at src/background_thread.c:255
|
#2 background_work_sleep_once (ind=0, info=<optimized out>, tsdn=<optimized out>)
|
at src/background_thread.c:307
|
#3 background_thread0_work (tsd=0x7f97e01fde20) at src/background_thread.c:452
|
#4 background_work (ind=<optimized out>, tsd=0x7f97e01fde20)
|
at src/background_thread.c:490
|
#5 background_thread_entry (ind_arg=<optimized out>) at src/background_thread.c:522
|
#6 0x00007f97e10d6dd5 in start_thread (arg=0x7f97e01ff700) at pthread_create.c:307
|
#7 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 31 (Thread 0x7f97df47e700 (LWP 21184)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97e4742edd in spdlog::details::thread_pool::process_next_msg_() ()
|
from /opt/couchbase/bin/../lib/libmemcached_logger.so.1.0.0
|
#2 0x00007f97e4743430 in spdlog::details::thread_pool::worker_loop_() ()
|
from /opt/couchbase/bin/../lib/libmemcached_logger.so.1.0.0
|
#3 0x00007f97e18bddcf in std::execute_native_thread_routine (__p=0x7f97e060a480)
|
at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83
|
#4 0x00007f97e10d6dd5 in start_thread (arg=0x7f97df47e700) at pthread_create.c:307
|
#5 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 30 (Thread 0x7f97dec7d700 (LWP 21185)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97e474251a in _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN6spdlog7details15periodic_workerC4ERKSt8functionIFvvEENSt6chrono8durationIlSt5ratioILl1ELl1EEEEEUlvE_EEEEE6_M_runEv () from /opt/couchbase/bin/../lib/libmemcached_logger.so.1.0.0
|
#2 0x00007f97e18bddcf in std::execute_native_thread_routine (__p=0x7f97e07fcb40)
|
at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83
|
#3 0x00007f97e10d6dd5 in start_thread (arg=0x7f97dec7d700) at pthread_create.c:307
|
#4 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 29 (Thread 0x7f97de47c700 (LWP 21186)):
|
#0 0x00007f97e0df520d in poll () at ../sysdeps/unix/syscall-template.S:81
|
#1 0x00000000004695ee in check_stdin_thread(void*) ()
|
#2 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#3 0x00007f97e10d6dd5 in start_thread (arg=0x7f97de47c700) at pthread_create.c:307
|
#4 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 28 (Thread 0x7f97ddc7b700 (LWP 21187)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x0000000000492d10 in ExternalAuthManagerThread::run() ()
|
---Type <return> to continue, or q <return> to quit---
|
#2 0x00007f97e36c176f in Couchbase::Thread::thread_entry() ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#3 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#4 0x00007f97e10d6dd5 in start_thread (arg=0x7f97ddc7b700) at pthread_create.c:307
|
#5 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 27 (Thread 0x7f97dd47a700 (LWP 21188)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00000000004e8dc6 in AuditImpl::consume_events() ()
|
#2 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#3 0x00007f97e10d6dd5 in start_thread (arg=0x7f97dd47a700) at pthread_create.c:307
|
#4 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 26 (Thread 0x7f97db538700 (LWP 21189)):
|
#0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38
|
#1 0x00007f97e299a31a in epoll_wait (epfd=<optimized out>, events=<optimized out>,
|
maxevents=<optimized out>, timeout=<optimized out>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll_sub.c:64
|
#2 0x00007f97e299a7b5 in epoll_dispatch (base=0x1df7960, tv=<optimized out>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:462
|
#3 0x00007f97e29924d2 in event_base_loop (base=0x1df7960, flags=0)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1947
|
#4 0x0000000000469cc9 in worker_libevent(void*) ()
|
#5 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#6 0x00007f97e10d6dd5 in start_thread (arg=0x7f97db538700) at pthread_create.c:307
|
#7 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 25 (Thread 0x7f97dad37700 (LWP 21190)):
|
#0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38
|
#1 0x00007f97e299a31a in epoll_wait (epfd=<optimized out>, events=<optimized out>,
|
maxevents=<optimized out>, timeout=<optimized out>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll_sub.c:64
|
#2 0x00007f97e299a7b5 in epoll_dispatch (base=0x1df7f50, tv=<optimized out>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:462
|
#3 0x00007f97e29924d2 in event_base_loop (base=0x1df7f50, flags=0)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1947
|
#4 0x0000000000469cc9 in worker_libevent(void*) ()
|
#5 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#6 0x00007f97e10d6dd5 in start_thread (arg=0x7f97dad37700) at pthread_create.c:307
|
---Type <return> to continue, or q <return> to quit---
|
#7 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 24 (Thread 0x7f97da536700 (LWP 21191)):
|
#0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38
|
#1 0x00007f97e299a31a in epoll_wait (epfd=<optimized out>, events=<optimized out>,
|
maxevents=<optimized out>, timeout=<optimized out>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll_sub.c:64
|
#2 0x00007f97e299a7b5 in epoll_dispatch (base=0x1df8580, tv=<optimized out>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:462
|
#3 0x00007f97e29924d2 in event_base_loop (base=0x1df8580, flags=0)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1947
|
#4 0x0000000000469cc9 in worker_libevent(void*) ()
|
#5 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#6 0x00007f97e10d6dd5 in start_thread (arg=0x7f97da536700) at pthread_create.c:307
|
#7 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 23 (Thread 0x7f97d9d35700 (LWP 21192)):
|
#0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38
|
#1 0x00007f97e299a31a in epoll_wait (epfd=<optimized out>, events=<optimized out>,
|
maxevents=<optimized out>, timeout=<optimized out>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll_sub.c:64
|
#2 0x00007f97e299a7b5 in epoll_dispatch (base=0x1df8cb0, tv=<optimized out>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:462
|
#3 0x00007f97e29924d2 in event_base_loop (base=0x1df8cb0, flags=0)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1947
|
#4 0x0000000000469cc9 in worker_libevent(void*) ()
|
#5 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#6 0x00007f97e10d6dd5 in start_thread (arg=0x7f97d9d35700) at pthread_create.c:307
|
#7 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 22 (Thread 0x7f97d9534700 (LWP 21193)):
|
#0 pthread_cond_wait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
|
#1 0x00007f97e18b81bc in __gthread_cond_wait (__mutex=<optimized out>,
|
__cond=<optimized out>)
|
at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864
|
#2 std::condition_variable::wait (this=<optimized out>, __lock=...)
|
at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53
|
#3 0x00000000004a0cf1 in Executor::run() ()
|
#4 0x00007f97e36c176f in Couchbase::Thread::thread_entry() ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
---Type <return> to continue, or q <return> to quit---
|
#5 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#6 0x00007f97e10d6dd5 in start_thread (arg=0x7f97d9534700) at pthread_create.c:307
|
#7 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 21 (Thread 0x7f97d8d33700 (LWP 21194)):
|
#0 pthread_cond_wait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
|
#1 0x00007f97e18b81bc in __gthread_cond_wait (__mutex=<optimized out>,
|
__cond=<optimized out>)
|
at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864
|
#2 std::condition_variable::wait (this=<optimized out>, __lock=...)
|
at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53
|
#3 0x00000000004a0cf1 in Executor::run() ()
|
#4 0x00007f97e36c176f in Couchbase::Thread::thread_entry() ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#5 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#6 0x00007f97e10d6dd5 in start_thread (arg=0x7f97d8d33700) at pthread_create.c:307
|
#7 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 20 (Thread 0x7f97d8532700 (LWP 21195)):
|
#0 pthread_cond_wait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
|
#1 0x00007f97e18b81bc in __gthread_cond_wait (__mutex=<optimized out>,
|
__cond=<optimized out>)
|
at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864
|
#2 std::condition_variable::wait (this=<optimized out>, __lock=...)
|
at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53
|
#3 0x00000000004a0cf1 in Executor::run() ()
|
#4 0x00007f97e36c176f in Couchbase::Thread::thread_entry() ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#5 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#6 0x00007f97e10d6dd5 in start_thread (arg=0x7f97d8532700) at pthread_create.c:307
|
#7 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 19 (Thread 0x7f97d7d31700 (LWP 21196)):
|
#0 pthread_cond_wait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
|
#1 0x00007f97e18b81bc in __gthread_cond_wait (__mutex=<optimized out>,
|
__cond=<optimized out>)
|
at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864
|
#2 std::condition_variable::wait (this=<optimized out>, __lock=...)
|
at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53
|
#3 0x00000000004a0cf1 in Executor::run() ()
|
#4 0x00007f97e36c176f in Couchbase::Thread::thread_entry() ()
|
---Type <return> to continue, or q <return> to quit---
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#5 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#6 0x00007f97e10d6dd5 in start_thread (arg=0x7f97d7d31700) at pthread_create.c:307
|
#7 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 18 (Thread 0x7f97d55fe700 (LWP 21209)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97dc3aed8f in MemoryTracker::statsThreadMainLoop(void*) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#2 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#3 0x00007f97e10d6dd5 in start_thread (arg=0x7f97d55fe700) at pthread_create.c:307
|
#4 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 17 (Thread 0x7f97d4a7d700 (LWP 21210)):
|
#0 pthread_cond_wait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
|
#1 0x00007f97e18b81bc in __gthread_cond_wait (__mutex=<optimized out>,
|
__cond=<optimized out>)
|
at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864
|
#2 std::condition_variable::wait (this=<optimized out>, __lock=...)
|
at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53
|
#3 0x00007f97db58a13c in magma::TaskWorker::loop(void*) ()
|
from /opt/couchbase/bin/../lib/../lib/../lib/libmagma.so
|
#4 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#5 0x00007f97e10d6dd5 in start_thread (arg=0x7f97d4a7d700) at pthread_create.c:307
|
#6 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 16 (Thread 0x7f97affff700 (LWP 21211)):
|
#0 pthread_cond_wait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
|
#1 0x00007f97e18b81bc in __gthread_cond_wait (__mutex=<optimized out>,
|
__cond=<optimized out>)
|
at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864
|
#2 std::condition_variable::wait (this=<optimized out>, __lock=...)
|
at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53
|
#3 0x00007f97db58a13c in magma::TaskWorker::loop(void*) ()
|
from /opt/couchbase/bin/../lib/../lib/../lib/libmagma.so
|
#4 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#5 0x00007f97e10d6dd5 in start_thread (arg=0x7f97affff700) at pthread_create.c:307
|
#6 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 15 (Thread 0x7f97af7fe700 (LWP 21212)):
|
#0 pthread_cond_wait@@GLIBC_2.3.2 ()
|
---Type <return> to continue, or q <return> to quit---
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
|
#1 0x00007f97e18b81bc in __gthread_cond_wait (__mutex=<optimized out>,
|
__cond=<optimized out>)
|
at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864
|
#2 std::condition_variable::wait (this=<optimized out>, __lock=...)
|
at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53
|
#3 0x00007f97db58a13c in magma::TaskWorker::loop(void*) ()
|
from /opt/couchbase/bin/../lib/../lib/../lib/libmagma.so
|
#4 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#5 0x00007f97e10d6dd5 in start_thread (arg=0x7f97af7fe700) at pthread_create.c:307
|
#6 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 14 (Thread 0x7f97aeffd700 (LWP 21213)):
|
#0 pthread_cond_wait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
|
#1 0x00007f97e18b81bc in __gthread_cond_wait (__mutex=<optimized out>,
|
__cond=<optimized out>)
|
at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864
|
#2 std::condition_variable::wait (this=<optimized out>, __lock=...)
|
at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53
|
#3 0x00007f97db58a13c in magma::TaskWorker::loop(void*) ()
|
from /opt/couchbase/bin/../lib/../lib/../lib/libmagma.so
|
#4 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#5 0x00007f97e10d6dd5 in start_thread (arg=0x7f97aeffd700) at pthread_create.c:307
|
#6 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 13 (Thread 0x7f97ae7fc700 (LWP 21214)):
|
#0 pthread_cond_wait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
|
#1 0x00007f97e18b81bc in __gthread_cond_wait (__mutex=<optimized out>,
|
__cond=<optimized out>)
|
at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864
|
#2 std::condition_variable::wait (this=<optimized out>, __lock=...)
|
at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53
|
#3 0x00007f97db58a13c in magma::TaskWorker::loop(void*) ()
|
from /opt/couchbase/bin/../lib/../lib/../lib/libmagma.so
|
#4 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#5 0x00007f97e10d6dd5 in start_thread (arg=0x7f97ae7fc700) at pthread_create.c:307
|
#6 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 12 (Thread 0x7f97adffb700 (LWP 21215)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97dc3ce2db in TaskQueue::_doSleep(ExecutorThread&, std::unique_lock<std::mutex>---Type <return> to continue, or q <return> to quit---
|
&) () from /opt/couchbase/bin/../lib/../lib/ep.so
|
#2 0x00007f97dc3cfa8b in TaskQueue::sleepThenFetchNextTask(ExecutorThread&) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#3 0x00007f97dc36fce7 in ExecutorPool::_nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#4 0x00007f97dc36fd8d in ExecutorPool::nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#5 0x00007f97dc37928f in ExecutorThread::run() ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#6 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#7 0x00007f97e10d6dd5 in start_thread (arg=0x7f97adffb700) at pthread_create.c:307
|
#8 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 11 (Thread 0x7f97ad7fa700 (LWP 21216)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97dc3ce2db in TaskQueue::_doSleep(ExecutorThread&, std::unique_lock<std::mutex>&) () from /opt/couchbase/bin/../lib/../lib/ep.so
|
#2 0x00007f97dc3cfa8b in TaskQueue::sleepThenFetchNextTask(ExecutorThread&) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#3 0x00007f97dc36fce7 in ExecutorPool::_nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#4 0x00007f97dc36fd8d in ExecutorPool::nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#5 0x00007f97dc37928f in ExecutorThread::run() ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#6 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#7 0x00007f97e10d6dd5 in start_thread (arg=0x7f97ad7fa700) at pthread_create.c:307
|
#8 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 10 (Thread 0x7f97acff9700 (LWP 21217)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97dc3ce2db in TaskQueue::_doSleep(ExecutorThread&, std::unique_lock<std::mutex>&) () from /opt/couchbase/bin/../lib/../lib/ep.so
|
#2 0x00007f97dc3cfa8b in TaskQueue::sleepThenFetchNextTask(ExecutorThread&) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#3 0x00007f97dc36fce7 in ExecutorPool::_nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#4 0x00007f97dc36fd8d in ExecutorPool::nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#5 0x00007f97dc37928f in ExecutorThread::run() ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#6 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#7 0x00007f97e10d6dd5 in start_thread (arg=0x7f97acff9700) at pthread_create.c:307
|
#8 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
---Type <return> to continue, or q <return> to quit---
|
Thread 9 (Thread 0x7f97a7fff700 (LWP 21218)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97dc3ce2db in TaskQueue::_doSleep(ExecutorThread&, std::unique_lock<std::mutex>&) () from /opt/couchbase/bin/../lib/../lib/ep.so
|
#2 0x00007f97dc3cfa8b in TaskQueue::sleepThenFetchNextTask(ExecutorThread&) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#3 0x00007f97dc36fce7 in ExecutorPool::_nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#4 0x00007f97dc36fd8d in ExecutorPool::nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#5 0x00007f97dc37928f in ExecutorThread::run() ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#6 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#7 0x00007f97e10d6dd5 in start_thread (arg=0x7f97a7fff700) at pthread_create.c:307
|
#8 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 8 (Thread 0x7f97a77fe700 (LWP 21219)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97dc3ce2db in TaskQueue::_doSleep(ExecutorThread&, std::unique_lock<std::mutex>&) () from /opt/couchbase/bin/../lib/../lib/ep.so
|
#2 0x00007f97dc3cfa8b in TaskQueue::sleepThenFetchNextTask(ExecutorThread&) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#3 0x00007f97dc36fce7 in ExecutorPool::_nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#4 0x00007f97dc36fd8d in ExecutorPool::nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#5 0x00007f97dc37928f in ExecutorThread::run() ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#6 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#7 0x00007f97e10d6dd5 in start_thread (arg=0x7f97a77fe700) at pthread_create.c:307
|
#8 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 7 (Thread 0x7f97a6ffd700 (LWP 21220)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97dc3ce2db in TaskQueue::_doSleep(ExecutorThread&, std::unique_lock<std::mutex>&) () from /opt/couchbase/bin/../lib/../lib/ep.so
|
#2 0x00007f97dc3cfa8b in TaskQueue::sleepThenFetchNextTask(ExecutorThread&) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#3 0x00007f97dc36fce7 in ExecutorPool::_nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#4 0x00007f97dc36fd8d in ExecutorPool::nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#5 0x00007f97dc37928f in ExecutorThread::run() ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#6 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
---Type <return> to continue, or q <return> to quit---
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#7 0x00007f97e10d6dd5 in start_thread (arg=0x7f97a6ffd700) at pthread_create.c:307
|
#8 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 6 (Thread 0x7f97a67fc700 (LWP 21221)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97dc3ce2db in TaskQueue::_doSleep(ExecutorThread&, std::unique_lock<std::mutex>&) () from /opt/couchbase/bin/../lib/../lib/ep.so
|
#2 0x00007f97dc3cfa8b in TaskQueue::sleepThenFetchNextTask(ExecutorThread&) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#3 0x00007f97dc36fce7 in ExecutorPool::_nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#4 0x00007f97dc36fd8d in ExecutorPool::nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#5 0x00007f97dc37928f in ExecutorThread::run() ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#6 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#7 0x00007f97e10d6dd5 in start_thread (arg=0x7f97a67fc700) at pthread_create.c:307
|
#8 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 5 (Thread 0x7f97a5ffb700 (LWP 21222)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97dc3ce2db in TaskQueue::_doSleep(ExecutorThread&, std::unique_lock<std::mutex>&) () from /opt/couchbase/bin/../lib/../lib/ep.so
|
#2 0x00007f97dc3cfa8b in TaskQueue::sleepThenFetchNextTask(ExecutorThread&) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#3 0x00007f97dc36fce7 in ExecutorPool::_nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#4 0x00007f97dc36fd8d in ExecutorPool::nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#5 0x00007f97dc37928f in ExecutorThread::run() ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#6 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#7 0x00007f97e10d6dd5 in start_thread (arg=0x7f97a5ffb700) at pthread_create.c:307
|
#8 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 4 (Thread 0x7f97a57fa700 (LWP 21223)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97dc3ce2db in TaskQueue::_doSleep(ExecutorThread&, std::unique_lock<std::mutex>&) () from /opt/couchbase/bin/../lib/../lib/ep.so
|
#2 0x00007f97dc3cfa8b in TaskQueue::sleepThenFetchNextTask(ExecutorThread&) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#3 0x00007f97dc36fce7 in ExecutorPool::_nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#4 0x00007f97dc36fd8d in ExecutorPool::nextTask(ExecutorThread&, unsigned char) ()
|
---Type <return> to continue, or q <return> to quit---
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#5 0x00007f97dc37928f in ExecutorThread::run() ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#6 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#7 0x00007f97e10d6dd5 in start_thread (arg=0x7f97a57fa700) at pthread_create.c:307
|
#8 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 3 (Thread 0x7f979ffff700 (LWP 21224)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97dc3ce2db in TaskQueue::_doSleep(ExecutorThread&, std::unique_lock<std::mutex>&) () from /opt/couchbase/bin/../lib/../lib/ep.so
|
#2 0x00007f97dc3cfa8b in TaskQueue::sleepThenFetchNextTask(ExecutorThread&) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#3 0x00007f97dc36fce7 in ExecutorPool::_nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#4 0x00007f97dc36fd8d in ExecutorPool::nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#5 0x00007f97dc37928f in ExecutorThread::run() ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#6 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#7 0x00007f97e10d6dd5 in start_thread (arg=0x7f979ffff700) at pthread_create.c:307
|
#8 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 2 (Thread 0x7f97a4ff9700 (LWP 21225)):
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 ()
|
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
|
#1 0x00007f97dc3ce2db in TaskQueue::_doSleep(ExecutorThread&, std::unique_lock<std::mutex>&) () from /opt/couchbase/bin/../lib/../lib/ep.so
|
#2 0x00007f97dc3cfa8b in TaskQueue::sleepThenFetchNextTask(ExecutorThread&) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#3 0x00007f97dc36fce7 in ExecutorPool::_nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#4 0x00007f97dc36fd8d in ExecutorPool::nextTask(ExecutorThread&, unsigned char) ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#5 0x00007f97dc37928f in ExecutorThread::run() ()
|
from /opt/couchbase/bin/../lib/../lib/ep.so
|
#6 0x00007f97e36b5ed7 in platform_thread_wrap(void*) ()
|
from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0
|
#7 0x00007f97e10d6dd5 in start_thread (arg=0x7f97a4ff9700) at pthread_create.c:307
|
#8 0x00007f97e0dffead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
|
|
|
Thread 1 (Thread 0x7f97e4d75040 (LWP 21182)):
|
#0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38
|
#1 0x00007f97e299a31a in epoll_wait (epfd=<optimized out>, events=<optimized out>,
|
maxevents=<optimized out>, timeout=<optimized out>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll_sub.c:64
|
---Type <return> to continue, or q <return> to quit---
|
#2 0x00007f97e299a7b5 in epoll_dispatch (base=0x1df4a50, tv=<optimized out>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:462
|
#3 0x00007f97e29924d2 in event_base_loop (base=0x1df4a50, flags=0)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1947
|
#4 0x0000000000429d18 in memcached_main ()
|
#5 0x00007f97e0d243d5 in __libc_start_main (main=0x41d090 <main>, argc=3,
|
argv=0x7ffc25302838, init=<optimized out>, fini=<optimized out>,
|
rtld_fini=<optimized out>, stack_end=0x7ffc25302828) at ../csu/libc-start.c:266
|
#6 0x0000000000420215 in _start ()
|
QE test:
gradle-site-plugin/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/ritesh.ini -p num_items=1000000,magma_storage=True -t rebalance_new.rebalance_in.RebalanceInTests.test_rebalance_in_with_ops,nodes_in=3 -p rerun=False,infra_log_level=debug,log_level=debug,skip_cleanup=True -m rest'
|