gdb /opt/couchbase/bin/memcached core.memcached.10757 GNU gdb (GDB) CentOS (7.0.1-45.el5.centos) Copyright (C) 2009 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting instructions, please see: ... Reading symbols from /opt/couchbase/bin/memcached...done. [New Thread 10761] [New Thread 14434] [New Thread 14433] [New Thread 14432] [New Thread 14431] [New Thread 12921] [New Thread 12883] [New Thread 12863] [New Thread 12538] [New Thread 12500] [New Thread 12480] [New Thread 12112] [New Thread 12074] [New Thread 12054] [New Thread 11732] [New Thread 11694] [New Thread 11674] [New Thread 11211] [New Thread 11190] [New Thread 11170] [New Thread 10800] [New Thread 10778] [New Thread 10764] [New Thread 10763] [New Thread 10762] [New Thread 10760] [New Thread 10759] [New Thread 10758] [New Thread 10757] Reading symbols from /opt/couchbase/bin/../lib/memcached/libmcd_util.so.1.0.0...done. Loaded symbols for /opt/couchbase/bin/../lib/memcached/libmcd_util.so.1.0.0 Reading symbols from /opt/couchbase/bin/../lib/libcbsasl.so.1.1.1...done. Loaded symbols for /opt/couchbase/bin/../lib/libcbsasl.so.1.1.1 Reading symbols from /opt/couchbase/bin/../lib/libplatform.so.0.1.0...done. Loaded symbols for /opt/couchbase/bin/../lib/libplatform.so.0.1.0 Reading symbols from /opt/couchbase/bin/../lib/libcJSON.so.1.0.0...done. Loaded symbols for /opt/couchbase/bin/../lib/libcJSON.so.1.0.0 Reading symbols from /opt/couchbase/bin/../lib/libJSON_checker.so...done. Loaded symbols for /opt/couchbase/bin/../lib/libJSON_checker.so Reading symbols from /opt/couchbase/bin/../lib/libsnappy.so.1...done. Loaded symbols for /opt/couchbase/bin/../lib/libsnappy.so.1 Reading symbols from /opt/couchbase/bin/../lib/libtcmalloc_minimal.so.4...done. Loaded symbols for /opt/couchbase/bin/../lib/libtcmalloc_minimal.so.4 Reading symbols from /opt/couchbase/bin/../lib/libevent_core-2.0.so.5...done. Loaded symbols for /opt/couchbase/bin/../lib/libevent_core-2.0.so.5 Reading symbols from /lib64/libssl.so.6...(no debugging symbols found)...done. Loaded symbols for /lib64/libssl.so.6 Reading symbols from /lib64/libcrypto.so.6...(no debugging symbols found)...done. Loaded symbols for /lib64/libcrypto.so.6 Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done. [Thread debugging using libthread_db enabled] Loaded symbols for /lib64/libpthread.so.0 Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done. Loaded symbols for /lib64/libdl.so.2 Reading symbols from /lib64/librt.so.1...(no debugging symbols found)...done. Loaded symbols for /lib64/librt.so.1 Reading symbols from /usr/lib64/libstdc++.so.6...(no debugging symbols found)...done. Loaded symbols for /usr/lib64/libstdc++.so.6 Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done. Loaded symbols for /lib64/libm.so.6 Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done. Loaded symbols for /lib64/libgcc_s.so.1 Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done. Loaded symbols for /lib64/libc.so.6 Reading symbols from /usr/lib64/libgssapi_krb5.so.2...(no debugging symbols found)...done. Loaded symbols for /usr/lib64/libgssapi_krb5.so.2 Reading symbols from /usr/lib64/libkrb5.so.3...(no debugging symbols found)...done. Loaded symbols for /usr/lib64/libkrb5.so.3 Reading symbols from /lib64/libcom_err.so.2...(no debugging symbols found)...done. Loaded symbols for /lib64/libcom_err.so.2 Reading symbols from /usr/lib64/libk5crypto.so.3...(no debugging symbols found)...done. Loaded symbols for /usr/lib64/libk5crypto.so.3 Reading symbols from /lib64/libz.so.1...(no debugging symbols found)...done. Loaded symbols for /lib64/libz.so.1 Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done. Loaded symbols for /lib64/ld-linux-x86-64.so.2 Reading symbols from /usr/lib64/libkrb5support.so.0...(no debugging symbols found)...done. Loaded symbols for /usr/lib64/libkrb5support.so.0 Reading symbols from /lib64/libkeyutils.so.1...(no debugging symbols found)...done. Loaded symbols for /lib64/libkeyutils.so.1 Reading symbols from /lib64/libresolv.so.2...(no debugging symbols found)...done. Loaded symbols for /lib64/libresolv.so.2 Reading symbols from /lib64/libselinux.so.1...(no debugging symbols found)...done. Loaded symbols for /lib64/libselinux.so.1 Reading symbols from /lib64/libsepol.so.1...(no debugging symbols found)...done. Loaded symbols for /lib64/libsepol.so.1 Reading symbols from /opt/couchbase/lib/memcached/stdin_term_handler.so...done. Loaded symbols for /opt/couchbase/lib/memcached/stdin_term_handler.so Reading symbols from /opt/couchbase/lib/memcached/file_logger.so...done. Loaded symbols for /opt/couchbase/lib/memcached/file_logger.so Reading symbols from /opt/couchbase/lib/memcached/bucket_engine.so...done. Loaded symbols for /opt/couchbase/lib/memcached/bucket_engine.so Reading symbols from /opt/couchbase/lib/memcached/ep.so...done. Loaded symbols for /opt/couchbase/lib/memcached/ep.so Reading symbols from /opt/couchbase/lib/libcouchstore.so...done. Loaded symbols for /opt/couchbase/lib/libcouchstore.so Reading symbols from /opt/couchbase/lib/libdirutils.so.0.1.0...done. Loaded symbols for /opt/couchbase/lib/libdirutils.so.0.1.0 Reading symbols from /opt/couchbase/lib/libv8.so...done. Loaded symbols for /opt/couchbase/lib/libv8.so Reading symbols from /opt/couchbase/lib/libicui18n.so.44...(no debugging symbols found)...done. Loaded symbols for /opt/couchbase/lib/libicui18n.so.44 Reading symbols from /opt/couchbase/lib/libicuuc.so.44...(no debugging symbols found)...done. Loaded symbols for /opt/couchbase/lib/libicuuc.so.44 Reading symbols from /opt/couchbase/lib/libicudata.so.44...(no debugging symbols found)...done. Loaded symbols for /opt/couchbase/lib/libicudata.so.44 warning: no loadable sections found in added symbol-file system-supplied DSO at 0x7fff5e7fd000 Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'. Program terminated with signal 11, Segmentation fault. #0 0x0000003ea5464d24 in std::_Rb_tree_rotate_left(std::_Rb_tree_node_base*, std::_Rb_tree_node_base*&) () from /usr/lib64/libstdc++.so.6 (gdb) t a a bt Thread 29 (Thread 0x2b426b7a3750 (LWP 10757)): #0 0x0000003ea28d4d98 in epoll_wait () from /lib64/libc.so.6 #1 0x00002b426b588376 in epoll_dispatch (base=0xb34a000, tv=) at epoll.c:404 #2 0x00002b426b573c44 in event_base_loop (base=0xb34a000, flags=) at event.c:1558 #3 0x000000000040f71c in main (argc=, argv=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/daemon/memcached.c:8768 Thread 28 (Thread 10758): #0 0x0000003ea28c6bab in read () from /lib64/libc.so.6 #1 0x0000003ea286cd77 in _IO_new_file_underflow () from /lib64/libc.so.6 #2 0x0000003ea286d73e in _IO_default_uflow_internal () from /lib64/libc.so.6 #3 0x0000003ea2862824 in _IO_getline_info_internal () from /lib64/libc.so.6 #4 0x0000003ea28616c9 in fgets () from /lib64/libc.so.6 #5 0x00002b426b7a48b1 in check_stdin_thread (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/extensions/daemon/stdin_check.c:38 #6 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #7 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #8 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 27 (Thread 10759): #0 0x0000003ea300b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002b426ab0497b in cb_cond_timedwait (cond=0x2aaaaacbee60, mutex=0x2aaaaacbee20, ms=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:156 #2 0x00002aaaaaaae548 in logger_thead_main (arg=0x6a96ac0) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/extensions/loggers/file_logger.c:372 #3 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #4 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #5 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 26 (Thread 10760): #0 0x0000003ea28d4d98 in epoll_wait () from /lib64/libc.so.6 #1 0x00002b426b588376 in epoll_dispatch (base=0xb34a280, tv=) at epoll.c:404 #2 0x00002b426b573c44 in event_base_loop (base=0xb34a280, flags=) at event.c:1558 #3 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #4 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #5 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 25 (Thread 10762): #0 0x0000003ea28d4d98 in epoll_wait () from /lib64/libc.so.6 #1 0x00002b426b588376 in epoll_dispatch (base=0xb34a780, tv=) at epoll.c:404 #2 0x00002b426b573c44 in event_base_loop (base=0xb34a780, flags=) at event.c:1558 #3 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #4 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #5 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 24 (Thread 10763): #0 0x0000003ea28d4d98 in epoll_wait () from /lib64/libc.so.6 #1 0x00002b426b588376 in epoll_dispatch (base=0xb34aa00, tv=) at epoll.c:404 #2 0x00002b426b573c44 in event_base_loop (base=0xb34aa00, flags=) at event.c:1558 #3 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #4 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #5 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 23 (Thread 10764): #0 0x0000003ea28d4d98 in epoll_wait () from /lib64/libc.so.6 #1 0x00002b426b588376 in epoll_dispatch (base=0xb34ac80, tv=) at epoll.c:404 #2 0x00002b426b573c44 in event_base_loop (base=0xb34ac80, flags=) at event.c:1558 ---Type to continue, or q to quit--- #3 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #4 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #5 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 22 (Thread 10778): #0 0x0000003ea289a901 in nanosleep () from /lib64/libc.so.6 #1 0x0000003ea28ce2f4 in usleep () from /lib64/libc.so.6 #2 0x00002aaaaaf45905 in updateStatsThread (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/memory_tracker.cc:36 #3 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #4 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #5 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 21 (Thread 10800): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0x6a86230) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 20 (Thread 11170): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0xb53cd80) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 19 (Thread 11190): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0xc8a9f80) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 18 (Thread 11211): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0xf3e7ab0) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 17 (Thread 11674): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0x113d9f00) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 16 (Thread 11694): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0x100c3800) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 15 (Thread 11732): ---Type to continue, or q to quit--- #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0x11d23810) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 14 (Thread 12054): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0xff45d40) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 13 (Thread 12074): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0xe3e56c0) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 12 (Thread 12112): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0xff45cd0) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 11 (Thread 12480): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0xbab7bf0) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 10 (Thread 12500): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0xf9ef570) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 9 (Thread 12538): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0x113d6230) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 8 (Thread 12863): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0x11aea380) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 ---Type to continue, or q to quit--- Thread 7 (Thread 12883): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0x12d81490) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 6 (Thread 12921): #0 0x0000003ea300af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaaacc3444 in engine_shutdown_thread (arg=0xfc76920) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1610 #2 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #3 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #4 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 5 (Thread 14431): #0 0x0000003ea300b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002b426ab0497b in cb_cond_timedwait (cond=0xb372e58, mutex=0xb372e20, ms=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:156 #2 0x00002aaaaaf36a5c in ExecutorPool::trySleep (this=0xb372e00, t=..., now=...) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/syncobject.h:74 #3 0x00002aaaaaf36c86 in ExecutorPool::nextTask (this=0xb372e00, t=..., tick=0 '\000') at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorpool.cc:153 #4 0x00002aaaaaf48088 in ExecutorThread::run (this=0xcf5ff00) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:77 #5 0x00002aaaaaf48656 in launch_executor_thread (arg=0xb372e5c) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:33 #6 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #7 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #8 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 4 (Thread 14432): #0 0x0000003ea300b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002b426ab0497b in cb_cond_timedwait (cond=0xb372e58, mutex=0xb372e20, ms=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:156 #2 0x00002aaaaaf36a5c in ExecutorPool::trySleep (this=0xb372e00, t=..., now=...) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/syncobject.h:74 #3 0x00002aaaaaf36c86 in ExecutorPool::nextTask (this=0xb372e00, t=..., tick=3 '\003') at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorpool.cc:153 #4 0x00002aaaaaf48088 in ExecutorThread::run (this=0xcf5cbd0) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:77 #5 0x00002aaaaaf48656 in launch_executor_thread (arg=0xb372e5c) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:33 #6 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #7 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #8 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 3 (Thread 14433): #0 0x0000003ea28cdeb7 in fdatasync () from /lib64/libc.so.6 #1 0x00002aaaab235637 in couch_sync (errinfo=0xecd5c20, handle=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/couchstore/src/os.c:133 #2 0x00002aaaaafb92eb in cfs_sync(._244 *, couch_file_handle) (errinfo=0xecd5c20, h=0xba69620) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-fs-stats.cc:128 #3 0x00002aaaab21f6fa in couchstore_commit (db=0xecd5c00) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/couchstore/src/couch_db.cc:194 #4 0x00002aaaaafaa419 in CouchKVStore::setVBucketState (this=0xb3d1a00, vbucketId=1023, vbstate=..., vb_change_type=1, kvcb=0x47959f70, notify=true) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1045 #5 0x00002aaaaafaabaf in CouchKVStore::snapshotVBuckets (this=0xb3d1a00, vbstates=Traceback (most recent call last): File "/usr/share/gdb/python/libstdcxx/v6/printers.py", line 288, in children nodetype = gdb.lookup_type('std::_Rb_tree_node< std::pair< %s, %s > >' % (keytype, valuetype)) RuntimeError: No type named std::_Rb_tree_node< std::pair< const unsigned short, vbucket_state > >. std::map with 256 elements, cb=0x47959f70) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:929 #6 0x00002aaaaaf07ecf in EventuallyPersistentStore::snapshotVBuckets (this=0xebad440, priority=..., shardId=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/ep.cc:851 #7 0x00002aaaaaf6c9ef in VBSnapshotTask::run (this=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/tasks.cc:44 #8 0x00002aaaaaf4811c in ExecutorThread::run (this=0xcf5c6c0) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:95 #9 0x00002aaaaaf48656 in launch_executor_thread (arg=0x4a) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:33 #10 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 ---Type to continue, or q to quit--- #11 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #12 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 2 (Thread 14434): #0 0x0000003ea300b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002b426ab0497b in cb_cond_timedwait (cond=0xb372e58, mutex=0xb372e20, ms=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:156 #2 0x00002aaaaaf36a5c in ExecutorPool::trySleep (this=0xb372e00, t=..., now=...) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/syncobject.h:74 #3 0x00002aaaaaf36c86 in ExecutorPool::nextTask (this=0xb372e00, t=..., tick=2 '\002') at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorpool.cc:153 #4 0x00002aaaaaf48088 in ExecutorThread::run (this=0xcf5cd80) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:77 #5 0x00002aaaaaf48656 in launch_executor_thread (arg=0xb372e5c) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:33 #6 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #7 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #8 0x0000003ea28d49ad in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x43353940 (LWP 10761)): #0 0x0000003ea5464d24 in std::_Rb_tree_rotate_left(std::_Rb_tree_node_base*, std::_Rb_tree_node_base*&) () from /usr/lib64/libstdc++.so.6 #1 0x0000003ea5464eb6 in std::_Rb_tree_insert_and_rebalance(bool, std::_Rb_tree_node_base*, std::_Rb_tree_node_base*, std::_Rb_tree_node_base&) () from /usr/lib64/libstdc++.so.6 #2 0x00002aaaaaf2e711 in std::_Rb_tree, std::_Select1st >, std::less, std::allocator > >::insert_unique (this=0xb3d1968, __position=, __v=...) at /usr/lib/gcc/x86_64-redhat-linux/4.1.2/../../../../include/c++/4.1.2/bits/stl_tree.h:821 #3 0x00002aaaaafa8bec in insert (this=0xb3d16c0, vbid=1022) at /usr/lib/gcc/x86_64-redhat-linux/4.1.2/../../../../include/c++/4.1.2/bits/stl_map.h:420 #4 operator[] (this=0xb3d16c0, vbid=1022) at /usr/lib/gcc/x86_64-redhat-linux/4.1.2/../../../../include/c++/4.1.2/bits/stl_map.h:348 #5 CouchKVStore::getNumPersistedDeletes (this=0xb3d16c0, vbid=1022) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:2146 #6 0x00002aaaaaf80380 in ActiveStream::addTakeoverStats (this=0x122c2160, add_stat=0x407df0 , cookie=0xb2ded00) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/upr-stream.cc:500 #7 0x00002aaaaaf781b8 in UprProducer::addTakeoverStats (this=0x122c34a0, add_stat=0x407df0 , c=0xb2ded00, vbid=1022) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/upr-producer.cc:396 #8 0x00002aaaaaf19c30 in EventuallyPersistentEngine::doUprVbTakeoverStats (this=, cookie=0xb2ded00, add_stat=0x407df0 , key= "replication:ns_1@10.3.5.73->ns_1@10.3.5.74:default", vbid=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/ep_engine.cc:5165 #9 0x00002aaaaaf28f23 in EventuallyPersistentEngine::getStats (this=0xcf96800, cookie=0xb2ded00, stat_key=, nkey=, add_stat=0x407df0 ) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/ep_engine.cc:4301 #10 0x00002aaaaaf29a47 in EvpGetStats (handle=0xcf96800, cookie=0xb2ded00, stat_key=0xb308018 "upr-vbtakeover 1022 replication:ns_1@10.3.5.73->ns_1@10.3.5.74:defaultcket_uuid=f2bf91bc6d1b62773e22feab54888b64\",\"streamingUri\":\"/pools/default/bucketsStreaming/default?bucket_uuid=f2bf91bc6d1b62773e"..., nkey=70, add_stat=0x407df0 ) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/ep_engine.cc:217 #11 0x00002aaaaacc62f0 in bucket_get_stats (handle=0x2aaaaaecc4a0, cookie=0xb2ded00, stat_key=, nkey=70, add_stat=0x407df0 ) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:1908 #12 0x00000000004140f6 in stat_executor (c=0xb2ded00, packet=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/daemon/memcached.c:5019 #13 0x000000000041832e in process_bin_packet (c=0xb2ded00) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/daemon/memcached.c:5390 #14 complete_nread (c=0xb2ded00) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/daemon/memcached.c:5796 #15 conn_nread (c=0xb2ded00) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/daemon/memcached.c:7003 #16 0x000000000040a37d in event_handler (fd=, which=, arg=0xb2ded00) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/daemon/memcached.c:7276 #17 0x00002b426b573d3c in event_process_active_single_queue (base=0xb34a500, flags=) at event.c:1308 #18 event_process_active (base=0xb34a500, flags=) at event.c:1375 #19 event_base_loop (base=0xb34a500, flags=) at event.c:1572 #20 0x00002b426ab04bff in platform_thread_wrap (arg=) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #21 0x0000003ea300677d in start_thread () from /lib64/libpthread.so.0 #22 0x0000003ea28d49ad in clone () from /lib64/libc.so.6