Details
-
Bug
-
Resolution: Fixed
-
Critical
-
5.0.0
-
Untriaged
-
-
No
Description
Crash occurred on xdcr cluster receiving mutation. This is a 2 node kv only cluster that is receiving a single xdcr stream (5=>2) during longevity testing. First time observing this crash:
Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'.
|
#0 vlength (this=<optimized out>, this=<optimized out>) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/blob.h:124 |
Thread 1 (LWP 26987): |
#0 vlength (this=<optimized out>, this=<optimized out>) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/blob.h:124 |
#1 VBucket::pruneXattrDocument (this=this@entry=0x7f45c85bbe00, v=..., itemMeta=...) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:2500 |
#2 0x00007f4622338382 in VBucket::deleteWithMeta (this=0x7f45c85bbe00, key=..., cas=@0x7f46195d26a0: 0, |
seqno=seqno@entry=0x7f46195d26c0, cookie=cookie@entry=0x7f461529d118, engine=..., bgFetchDelay=0, |
checkConflicts=checkConflicts@entry=Yes, itemMeta=..., backfill=backfill@entry=false, genBySeqno=genBySeqno@entry=Yes, |
generateCas=generateCas@entry=No, bySeqno=bySeqno@entry=0, isReplication=isReplication@entry=false) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:1354 |
#3 0x00007f462230967e in KVBucket::deleteWithMeta (this=0x7f461024e000, key=..., cas=@0x7f46195d26a0: 0, seqno=0x7f46195d26c0, |
vbucket=<optimized out>, cookie=0x7f461529d118, permittedVBStates=..., checkConflicts=Yes, itemMeta=..., backfill=false, |
genBySeqno=Yes, generateCas=No, bySeqno=0, emd=0x0, isReplication=false) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:1883 |
#4 0x00007f46222c1fa4 in EventuallyPersistentEngine::deleteWithMeta (this=this@entry=0x7f4624d89800, vbucket=vbucket@entry=213, |
key=..., itemMeta=..., cas=@0x7f46195d26a0: 0, seqno=seqno@entry=0x7f46195d26c0, cookie=cookie@entry=0x7f461529d118, |
permittedVBStates=..., checkConflicts=Yes, genBySeqno=genBySeqno@entry=Yes, genCas=No, emd=...) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:4569 |
#5 0x00007f46222cad8a in EventuallyPersistentEngine::deleteWithMeta (this=this@entry=0x7f4624d89800, |
cookie=cookie@entry=0x7f461529d118, request=request@entry=0x7f43c3a1e5e9, response=response@entry= |
0x485ba0 <mcbp_response_handler(void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, unsigned short, unsigned long, void const*)>, docNamespace=docNamespace@entry=DefaultCollection) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:4500 |
#6 0x00007f46222d17b4 in processUnknownCommand (docNamespace=DefaultCollection, |
response=0x485ba0 <mcbp_response_handler(void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, unsigned short, unsigned long, void const*)>, request=0x7f43c3a1e5e9, cookie=0x7f461529d118, h=0x7f4624d89800) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:1173 |
#7 EvpUnknownCommand (handle=0x7f4624d89800, cookie=0x7f461529d118, request=0x7f43c3a1e5e9, |
response=0x485ba0 <mcbp_response_handler(void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, unsigned short, unsigned long, void const*)>, doc_namespace=<optimized out>) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:1270 |
#8 0x000000000042065c in bucket_unknown_command (c=0x7f461529ce00, request=request@entry=0x7f43c3a1e5e9, |
response=response@entry=0x485ba0 <mcbp_response_handler(void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, unsigned short, unsigned long, void const*)>) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:28 |
#9 0x000000000044e5f4 in default_unknown_command (void_cookie=<optimized out>, request=0x7f43c3a1e5e9, |
response=0x485ba0 <mcbp_response_handler(void const*, unsigned short, void const*, unsigned char, void const*, unsigned int, unsigned char, unsigned short, unsigned long, void const*)>) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:204 |
#10 0x0000000000453a58 in process_bin_unknown_packet (packet=0x7f43c3a1e5e9, c=0x7f461529ce00) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:241 |
#11 execute_request_packet (c=0x7f461529ce00) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:1372 |
#12 mcbp_complete_packet (c=c@entry=0x7f461529ce00) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:1483 |
#13 0x0000000000474e8f in conn_execute (c=0x7f461529ce00) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/statemachine_mcbp.cc:306 |
#14 0x0000000000441d38 in execute (connection=..., this=<optimized out>) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/statemachine_mcbp.h:43 |
#15 McbpConnection::runStateMachinery (this=0x7f461529ce00) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection_mcbp.cc:870 |
#16 0x0000000000441d7b in McbpConnection::runEventLoop (this=0x7f461529ce00, which=-16896) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection_mcbp.cc:1169 |
#17 0x00000000004459ae in run_event_loop (c=0x7f461529ce00, which=<optimized out>) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connections.cc:146 |
#18 0x00000000004309f6 in thread_libevent_process (fd=<optimized out>, which=<optimized out>, arg=0x7f4624d82f78) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:337 |
#19 0x00007f46280d47ec in event_process_active_single_queue.isra.26 () from /opt/couchbase/lib/libevent_core.so.2.1.8 |
#20 0x00007f46280d7c3f in event_base_loop () from /opt/couchbase/lib/libevent_core.so.2.1.8 |
#21 0x0000000000430524 in worker_libevent (arg=0x7f4624d82f78) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:233 |
#22 0x00007f4628fb40ae in run (this=0x7f4624d2fbe0) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:59 |
#23 platform_thread_wrap (arg=0x7f4624d2fbe0) |
at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:72 |
#24 0x00007f4628974dc5 in start_thread () from /usr/lib64/libpthread-2.17.so |
#25 0x00007f4626b2428d in clone () from /usr/lib64/libc-2.17.so |
The crash was seen on both of the remote nodes.