Details
-
Bug
-
Resolution: Duplicate
-
Critical
-
3.0
-
Security Level: Public
-
None
-
centos 6.4 64-bit
-
Triaged
-
Centos 64-bit
-
-
Yes
Description
test rebalance.rebalancein.RebalanceInTests.rebalance_in_with_ops -p nodes_in=3,items=0,default_bucket=false,standard_buckets=5,sasl_buckets=5 failed in build 3.0.0-901 in sanity test.
Re-run one more time with same build and rebalance failed again with error
<0.8500.15> exited with {unexpected_exit,
{'EXIT',<0.9675.15>,
{wait_seqno_persisted_failed,"standard_bucket4",
383,0,
[{'ns_1@172.23.106.140',
{'EXIT',
{{{{{child_interrupted,
{'EXIT',<13597.26102.3>,normal}},
[{upr_replicator,spawn_and_wait,1,
[
{upr_replicator,handle_call,3,
[{file,"src/upr_replicator.erl"}
,
{line,112}]},
{gen_server,handle_msg,5,
[
,
{line,585}]},
{proc_lib,init_p_do_apply,3,
[
,
{line,239}]}]},
{gen_server,call,
['upr_replicator-standard_bucket4-ns_1@172.23.106.139',
,
infinity]}},
{gen_server,call,
['replication_manager-standard_bucket4',
,
infinity]}},
{gen_server,call,
[
,
{if_rebalance,<0.26322.14>,
{wait_seqno_persisted,383,0}},
infinity]}}}}]}}} ns_vbucket_mover000
And memcached crashed on node 139
Back trace from memcached coredump
Reading symbols from /opt/couchbase/lib/libcouchstore.so...done.
Loaded symbols for /opt/couchbase/lib/libcouchstore.so
Reading symbols from /opt/couchbase/lib/libdirutils.so.0.1.0...done.
Loaded symbols for /opt/couchbase/lib/libdirutils.so.0.1.0
Reading symbols from /opt/couchbase/lib/libv8.so...done.
Loaded symbols for /opt/couchbase/lib/libv8.so
Reading symbols from /opt/couchbase/lib/libicui18n.so.44...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/lib/libicui18n.so.44
Reading symbols from /opt/couchbase/lib/libicuuc.so.44...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/lib/libicuuc.so.44
Reading symbols from /opt/couchbase/lib/libicudata.so.44...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/lib/libicudata.so.44
warning: no loadable sections found in added symbol-file system-supplied DSO at 0x7fff8c53e000
Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'.
Program terminated with signal 6, Aborted.
#0 0x0000003fede30265 in raise () from /lib64/libc.so.6
Thread 13 (Thread 0x2aace35ad750 (LWP 4430)):
#0 0x0000003feded5658 in epoll_wait () from /lib64/libc.so.6
#1 0x00002aace3392376 in epoll_dispatch (base=0xea1a000, tv=<value optimized out>) at epoll.c:404
#2 0x00002aace337dc44 in event_base_loop (base=0xea1a000, flags=<value optimized out>) at event.c:1558
#3 0x000000000040f71c in main (argc=<value optimized out>, argv=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/daemon/memcached.c:8768
Thread 12 (Thread 4431):
#0 0x0000003fedec748b in read () from /lib64/libc.so.6
#1 0x0000003fede6bfa7 in _IO_new_file_underflow () from /lib64/libc.so.6
#2 0x0000003fede6c96e in _IO_default_uflow_internal () from /lib64/libc.so.6
#3 0x0000003fede61954 in _IO_getline_info_internal () from /lib64/libc.so.6
#4 0x0000003fede607f9 in fgets () from /lib64/libc.so.6
#5 0x00002aace35ae8b1 in check_stdin_thread (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/extensions/daemon/stdin_check.c:38
#6 0x00002aace290ebff in platform_thread_wrap (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#7 0x0000003fee60683d in start_thread () from /lib64/libpthread.so.0
#8 0x0000003feded526d in clone () from /lib64/libc.so.6
Thread 11 (Thread 4432):
#0 0x0000003fee60b280 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aace290e97b in cb_cond_timedwait (cond=0x2aace43c3e60, mutex=0x2aace43c3e20, ms=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:156
#2 0x00002aace41b3548 in logger_thead_main (arg=0xa166ac0) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/extensions/loggers/file_logger.c:372
#3 0x00002aace290ebff in platform_thread_wrap (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x0000003fee60683d in start_thread () from /lib64/libpthread.so.0
#5 0x0000003feded526d in clone () from /lib64/libc.so.6
Thread 10 (Thread 4433):
#0 0x0000003fee60d654 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003fee608f4a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x0000003fee608e0c in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00002aace290ea89 in cb_mutex_enter (mutex=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:85
#4 0x00002aace8c5357d in Mutex::acquire (this=0xf126760) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/mutex.cc:31
#5 0x00002aace8c8956e in lock (this=0xf126700) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/locks.h:66
#6 LockHolder (this=0xf126700) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/locks.h:44
#7 PassiveStream::next (this=0xf126700) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/upr-stream.cc:1118
#8 0x00002aace8c7f24b in UprConsumer::getNextItem (this=0xf247500) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/upr-consumer.cc:593
#9 0x00002aace8c7f42f in UprConsumer::step (this=0xf247500, producers=0x6222a0) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/upr-consumer.cc:381
#10 0x00002aace8c1ef52 in EvpUprStep (handle=<value optimized out>, cookie=0xe8d0100, producers=0x6222a0) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/ep_engine.cc:1350
#11 0x00002aace4dce1d6 in upr_step (handle=<value optimized out>, cookie=0xe8d0100, producers=0x6222a0) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/engines/bucket_engine/bucket_engine.c:2225
#12 0x000000000041347f in ship_upr_log (c=0xe8d0100) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/daemon/memcached.c:3480
#13 conn_ship_log (c=0xe8d0100) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/daemon/memcached.c:6833
#14 0x000000000040a37d in event_handler (fd=<value optimized out>, which=<value optimized out>, arg=0xe8d0100) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/memcached/daemon/memcached.c:7276
#15 0x00002aace337dd3c in event_process_active_single_queue (base=0xea1a280, flags=<value optimized out>) at event.c:1308
#16 event_process_active (base=0xea1a280, flags=<value optimized out>) at event.c:1375
#17 event_base_loop (base=0xea1a280, flags=<value optimized out>) at event.c:1572
#18 0x00002aace290ebff in platform_thread_wrap (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#19 0x0000003fee60683d in start_thread () from /lib64/libpthread.so.0
#20 0x0000003feded526d in clone () from /lib64/libc.so.6
Thread 9 (Thread 4434):
#0 0x0000003feded5658 in epoll_wait () from /lib64/libc.so.6
#1 0x00002aace3392376 in epoll_dispatch (base=0xea1a500, tv=<value optimized out>) at epoll.c:404
#2 0x00002aace337dc44 in event_base_loop (base=0xea1a500, flags=<value optimized out>) at event.c:1558
#3 0x00002aace290ebff in platform_thread_wrap (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x0000003fee60683d in start_thread () from /lib64/libpthread.so.0
#5 0x0000003feded526d in clone () from /lib64/libc.so.6
Thread 8 (Thread 4435):
#0 0x0000003feded5658 in epoll_wait () from /lib64/libc.so.6
#1 0x00002aace3392376 in epoll_dispatch (base=0xea1a780, tv=<value optimized out>) at epoll.c:404
#2 0x00002aace337dc44 in event_base_loop (base=0xea1a780, flags=<value optimized out>) at event.c:1558
#3 0x00002aace290ebff in platform_thread_wrap (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x0000003fee60683d in start_thread () from /lib64/libpthread.so.0
#5 0x0000003feded526d in clone () from /lib64/libc.so.6
Thread 7 (Thread 4436):
#0 0x0000003feded5658 in epoll_wait () from /lib64/libc.so.6
#1 0x00002aace3392376 in epoll_dispatch (base=0xea1aa00, tv=<value optimized out>) at epoll.c:404
#2 0x00002aace337dc44 in event_base_loop (base=0xea1aa00, flags=<value optimized out>) at event.c:1558
#3 0x00002aace290ebff in platform_thread_wrap (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x0000003fee60683d in start_thread () from /lib64/libpthread.so.0
#5 0x0000003feded526d in clone () from /lib64/libc.so.6
Thread 6 (Thread 4437):
#0 0x0000003feded5658 in epoll_wait () from /lib64/libc.so.6
#1 0x00002aace3392376 in epoll_dispatch (base=0xea1ac80, tv=<value optimized out>) at epoll.c:404
#2 0x00002aace337dc44 in event_base_loop (base=0xea1ac80, flags=<value optimized out>) at event.c:1558
#3 0x00002aace290ebff in platform_thread_wrap (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x0000003fee60683d in start_thread () from /lib64/libpthread.so.0
#5 0x0000003feded526d in clone () from /lib64/libc.so.6
Thread 5 (Thread 12755):
#0 0x0000003fede9a4a1 in nanosleep () from /lib64/libc.so.6
#1 0x0000003fedecebb4 in usleep () from /lib64/libc.so.6
#2 0x00002aace8c52155 in updateStatsThread (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/memory_tracker.cc:36
#3 0x00002aace290ebff in platform_thread_wrap (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x0000003fee60683d in start_thread () from /lib64/libpthread.so.0
#5 0x0000003feded526d in clone () from /lib64/libc.so.6
Thread 4 (Thread 12757):
#0 0x0000003fedece777 in fdatasync () from /lib64/libc.so.6
#1 0x00002aace8f42637 in couch_sync (errinfo=0xa176520, handle=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/couchstore/src/os.c:133
#2 0x00002aace8cc5a2b in cfs_sync(._244 *, couch_file_handle) (errinfo=0xa176520, h=0xebcd5a0) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-fs-stats.cc:128
#3 0x00002aace8f2c72a in couchstore_commit (db=0xa176500) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/couchstore/src/couch_db.cc:203
#4 0x00002aace8cba7b9 in CouchKVStore::setVBucketState (this=0xea9c000, vbucketId=372, vbstate=..., vb_change_type=1, kvcb=0x2aacec87af70, notify=true)
at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1048
#5 0x00002aace8cbaf57 in CouchKVStore::snapshotVBuckets (this=0xea9c000, vbstates=Traceback (most recent call last):
File "/usr/share/gdb/python/libstdcxx/v6/printers.py", line 288, in children
nodetype = gdb.lookup_type('std::_Rb_tree_node< std::pair< %s, %s > >' % (keytype, valuetype))
RuntimeError: No type named std::_Rb_tree_node< std::pair< const unsigned short, vbucket_state > >.
std::map with 69 elements, cb=0x2aacec87af70) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:932
#6 0x00002aace8c0cb0f in EventuallyPersistentStore::snapshotVBuckets (this=0xea34240, priority=..., shardId=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/ep.cc:851
#7 0x00002aace8c7924f in VBSnapshotTask::run (this=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/tasks.cc:44
#8 0x00002aace8c5496c in ExecutorThread::run (this=0xa1ae3f0) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:95
#9 0x00002aace8c54ea6 in launch_executor_thread (arg=0x80) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:33
#10 0x00002aace290ebff in platform_thread_wrap (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#11 0x0000003fee60683d in start_thread () from /lib64/libpthread.so.0
#12 0x0000003feded526d in clone () from /lib64/libc.so.6
Thread 3 (Thread 12758):
#0 0x0000003fee60b280 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aace290e97b in cb_cond_timedwait (cond=0xebc8058, mutex=0xebc8020, ms=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:156
#2 0x00002aace8c42f5c in ExecutorPool::trySleep (this=0xebc8000, t=..., now=...) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/syncobject.h:74
#3 0x00002aace8c4316b in ExecutorPool::_nextTask (this=0xebc8000, t=..., tick=1 '\001') at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorpool.cc:139
#4 0x00002aace8c431de in ExecutorPool::nextTask (this=0xebc8000, t=..., tick=125 '}') at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorpool.cc:165
#5 0x00002aace8c548d8 in ExecutorThread::run (this=0xa1ae480) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:77
#6 0x00002aace8c54ea6 in launch_executor_thread (arg=0xebc805c) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:33
#7 0x00002aace290ebff in platform_thread_wrap (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#8 0x0000003fee60683d in start_thread () from /lib64/libpthread.so.0
#9 0x0000003feded526d in clone () from /lib64/libc.so.6
Thread 2 (Thread 12759):
#0 0x0000003fedece777 in fdatasync () from /lib64/libc.so.6
#1 0x00002aace8f42637 in couch_sync (errinfo=0xf4837a0, handle=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/couchstore/src/os.c:133
#2 0x00002aace8cc5a2b in cfs_sync(._244 *, couch_file_handle) (errinfo=0xf4837a0, h=0xebc1c20) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-fs-stats.cc:128
#3 0x00002aace8f2c6fa in couchstore_commit (db=0xf483780) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/couchstore/src/couch_db.cc:194
#4 0x00002aace8cba7b9 in CouchKVStore::setVBucketState (this=0xea9dd40, vbucketId=382, vbstate=..., vb_change_type=1, kvcb=0x2aacedc7cf70, notify=true)
at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1048
#5 0x00002aace8cbaf57 in CouchKVStore::snapshotVBuckets (this=0xea9dd40, vbstates=Traceback (most recent call last):
File "/usr/share/gdb/python/libstdcxx/v6/printers.py", line 288, in children
nodetype = gdb.lookup_type('std::_Rb_tree_node< std::pair< %s, %s > >' % (keytype, valuetype))
RuntimeError: No type named std::_Rb_tree_node< std::pair< const unsigned short, vbucket_state > >.
std::map with 68 elements, cb=0x2aacedc7cf70) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:932
#6 0x00002aace8c0cb0f in EventuallyPersistentStore::snapshotVBuckets (this=0xea34240, priority=..., shardId=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/ep.cc:851
#7 0x00002aace8c7924f in VBSnapshotTask::run (this=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/tasks.cc:44
#8 0x00002aace8c5496c in ExecutorThread::run (this=0xa1ae510) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:95
#9 0x00002aace8c54ea6 in launch_executor_thread (arg=0x4f) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:33
#10 0x00002aace290ebff in platform_thread_wrap (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#11 0x0000003fee60683d in start_thread () from /lib64/libpthread.so.0
#12 0x0000003feded526d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x2aacebe7a940 (LWP 12756)):
#0 0x0000003fede30265 in raise () from /lib64/libc.so.6
#1 0x0000003fede31d10 in abort () from /lib64/libc.so.6
#2 0x0000003fede29706 in __assert_fail () from /lib64/libc.so.6
#3 0x00002aace8c8e952 in PassiveStream::~PassiveStream (this=0xf126700, __in_chrg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/upr-stream.cc:856
#4 0x00002aace8c80545 in UprConsumer::processBufferedItems (this=0xf247500) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/atomic.h:397
#5 0x00002aace8c805fa in Processer::run (this=0xa1af200) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/upr-consumer.cc:48
#6 0x00002aace8c5496c in ExecutorThread::run (this=0xa1ae360) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:95
#7 0x00002aace8c54ea6 in launch_executor_thread (arg=0x114e) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/ep-engine/src/executorthread.cc:33
#8 0x00002aace290ebff in platform_thread_wrap (arg=<value optimized out>) at /buildbot/build_slave/centos-5-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#9 0x0000003fee60683d in start_thread () from /lib64/libpthread.so.0
#10 0x0000003feded526d in clone () from /lib64/libc.so.6
--------------------------------------------------------------------------------