memcached crash in release_handle_locked at bucket_engine.c:597

possibly after bucket deletion in the test cleanup.

Core was generated by `/opt/membase/bin/memcached -X /opt/membase/lib/memcached/stdin_term_handler.so'.
Program terminated with signal 6, Aborted.
#0 0x00007f1f4cd82a75 in raise () from /lib/libc.so.6
(gdb) t a a bt

Thread 12 (Thread 16279):
#0 0x00007f1f4ce2e31d in fdatasync () from /lib/libc.so.6
#1 0x00007f1f48e6b5c9 in full_fsync (id=0x7f1f380bf160, flags=2) at embedded/sqlite3.c:25510
#2 unixSync (id=0x7f1f380bf160, flags=2) at embedded/sqlite3.c:25558
#3 0x00007f1f48e8ad6d in sqlite3OsSync (pWal=0x7f1f3801c4c8, sync_flags=2, nBuf=1024, zBuf=0x7f1f380c1eb8 "")
at embedded/sqlite3.c:13413
#4 walCheckpoint (pWal=0x7f1f3801c4c8, sync_flags=2, nBuf=1024, zBuf=0x7f1f380c1eb8 "") at embedded/sqlite3.c:42447
#5 sqlite3WalCheckpoint (pWal=0x7f1f3801c4c8, sync_flags=2, nBuf=1024, zBuf=0x7f1f380c1eb8 "")
at embedded/sqlite3.c:43431
#6 0x00007f1f48ea48fe in sqlite3PagerCheckpoint (db=0x7f1f3806b5b8, zDb=<value optimized out>)
at embedded/sqlite3.c:40675
#7 sqlite3BtreeCheckpoint (db=0x7f1f3806b5b8, zDb=<value optimized out>) at embedded/sqlite3.c:52433
#8 sqlite3Checkpoint (db=0x7f1f3806b5b8, zDb=<value optimized out>) at embedded/sqlite3.c:39117
#9 sqlite3_wal_checkpoint (db=0x7f1f3806b5b8, zDb=<value optimized out>) at embedded/sqlite3.c:39082
#10 0x00007f1f48ea4a89 in sqlite3WalDefaultHook (pClientData=<value optimized out>, db=0x7f1f3806b5b8,
zDb=0x7f1f380a3760 "kv_0", nFrame=-1) at embedded/sqlite3.c:104546
#11 0x00007f1f48ede307 in doWalCallbacks (pStmt=<value optimized out>) at embedded/sqlite3.c:57877
#12 sqlite3Step (pStmt=<value optimized out>) at embedded/sqlite3.c:57962
#13 sqlite3_step (pStmt=<value optimized out>) at embedded/sqlite3.c:58011
#14 0x00007f1f48e5c45a in PreparedStatement::execute (this=0x7f1f43ffe2e0) at sqlite-pst.cc:73
#15 0x00007f1f48e5e018 in SqliteStrategy::execute (this=<value optimized out>, query=<value optimized out>)
at sqlite-strategies.cc:151
#16 0x00007f1f48e5e5bb in ShardedByVBucketSqliteStrategy::destroyTables (this=0x7f1f440614d0)
at sqlite-strategies.cc:417
#17 0x00007f1f48e5aa5f in StrategicSqlite3::reset (this=0x7f1f44061590) at sqlite-kvstore.cc:145
#18 0x00007f1f48e0153d in EventuallyPersistentStore::flushOneDeleteAll (this=0x7f1f44534e60, q=0x7f1f44534f90,
rejectQueue=0x7f1f380dcef0) at ep.cc:1784
#19 EventuallyPersistentStore::flushOne (this=0x7f1f44534e60, q=0x7f1f44534f90, rejectQueue=0x7f1f380dcef0)
at ep.cc:1923
#20 0x00007f1f48e0179a in EventuallyPersistentStore::flushSome (this=0x7f1f44534e60, q=0x7f1f44534f90,
rejectQueue=0x7f1f380dcef0) at ep.cc:1535
--Type <return> to continue, or q <return> to quit--
#21 0x00007f1f48e3165d in Flusher::doFlush (this=0x7f1f46036720) at flusher.cc:242
#22 0x00007f1f48e320d5 in Flusher::step (this=0x2d, d=..., tid=...) at flusher.cc:156
#23 0x00007f1f48e3260a in FlusherStepper::callback (this=<value optimized out>, d=..., t=<value optimized out>)
at flusher.cc:23
#24 0x00007f1f48df3e8a in Task::run (this=<value optimized out>, d=..., t=<value optimized out>) at dispatcher.hh:139
#25 0x00007f1f48df2c05 in Dispatcher::run (this=0x7f1f450f5070) at dispatcher.cc:119
#26 0x00007f1f48df3813 in launch_dispatcher_thread (arg=0x2d) at dispatcher.cc:28
#27 0x00007f1f4d0d79ca in start_thread () from /lib/libpthread.so.0
#28 0x00007f1f4ce356cd in clone () from /lib/libc.so.6
#29 0x0000000000000000 in ?? ()

Thread 11 (Thread 16275):
#0 0x00007f1f4ce35cc3 in epoll_wait () from /lib/libc.so.6
#1 0x00007f1f4d9a095c in epoll_dispatch (base=0xacd2c0, tv=<value optimized out>) at epoll.c:404
#2 0x00007f1f4d98f2cc in event_base_loop (base=0xacd2c0, flags=<value optimized out>) at event.c:1558
#3 0x0000000000414824 in worker_libevent (arg=0xacbe20) at daemon/thread.c:304
#4 0x00007f1f4d0d79ca in start_thread () from /lib/libpthread.so.0
#5 0x00007f1f4ce356cd in clone () from /lib/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 10 (Thread 16276):
#0 0x00007f1f4ce35cc3 in epoll_wait () from /lib/libc.so.6
#1 0x00007f1f4d9a095c in epoll_dispatch (base=0xacdb60, tv=<value optimized out>) at epoll.c:404
#2 0x00007f1f4d98f2cc in event_base_loop (base=0xacdb60, flags=<value optimized out>) at event.c:1558
#3 0x0000000000414824 in worker_libevent (arg=0xacbf20) at daemon/thread.c:304
#4 0x00007f1f4d0d79ca in start_thread () from /lib/libpthread.so.0
#5 0x00007f1f4ce356cd in clone () from /lib/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 9 (Thread 16270):
#0 0x00007f1f4ce35cc3 in epoll_wait () from /lib/libc.so.6
#1 0x00007f1f4d9a095c in epoll_dispatch (base=0xac0b40, tv=<value optimized out>) at epoll.c:404
--Type <return> to continue, or q <return> to quit--
#2 0x00007f1f4d98f2cc in event_base_loop (base=0xac0b40, flags=<value optimized out>) at event.c:1558
#3 0x000000000040ae28 in main (argc=<value optimized out>, argv=0x7ffff539e658) at daemon/memcached.c:7459

Thread 8 (Thread 16271):
#0 0x00007f1f4ce2747d in read () from /lib/libc.so.6
#1 0x00007f1f4cdc2348 in _IO_file_underflow () from /lib/libc.so.6
#2 0x00007f1f4cdc3eee in _IO_default_uflow () from /lib/libc.so.6
#3 0x00007f1f4cdbac7b in getc () from /lib/libc.so.6
#4 0x00007f1f4cb4d965 in check_stdin_thread (arg=0x403b70) at extensions/daemon/stdin_check.c:19
#5 0x00007f1f4d0d79ca in start_thread () from /lib/libpthread.so.0
#6 0x00007f1f4ce356cd in clone () from /lib/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 7 (Thread 16372):
#0 0x00007f1f4d0d903d in pthread_join () from /lib/libpthread.so.0
#1 0x00007f1f48df1028 in Dispatcher::stop (this=0x7f1f450f5070, force=true) at dispatcher.cc:158
#2 0x00007f1f48df70c5 in ~EventuallyPersistentStore (this=0x7f1f44534e60, __in_chrg=<value optimized out>) at ep.cc:418

#3 0x00007f1f48e155af in ~EventuallyPersistentEngine (handle=0x7f1f440586e0, force=<value optimized out>)
at ep_engine.h:510
#4 EvpDestroy (handle=0x7f1f440586e0, force=<value optimized out>) at ep_engine.cc:97
#5 0x00007f1f4b942f9f in engine_shutdown_thread (arg=0x7f1f4404c430) at bucket_engine.c:1394
#6 0x00007f1f4d0d79ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f1f4ce356cd in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 6 (Thread 16273):
#0 0x00007f1f4ce35cc3 in epoll_wait () from /lib/libc.so.6
#1 0x00007f1f4d9a095c in epoll_dispatch (base=0xacc180, tv=<value optimized out>) at epoll.c:404
#2 0x00007f1f4d98f2cc in event_base_loop (base=0xacc180, flags=<value optimized out>) at event.c:1558
#3 0x0000000000414824 in worker_libevent (arg=0xacbc20) at daemon/thread.c:304
#4 0x00007f1f4d0d79ca in start_thread () from /lib/libpthread.so.0
#5 0x00007f1f4ce356cd in clone () from /lib/libc.so.6
--Type <return> to continue, or q <return> to quit--
#6 0x0000000000000000 in ?? ()

Thread 5 (Thread 16278):
#0 0x00007f1f4d0dc85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007f1f48df2e6d in SyncObject::wait (this=0x7f1f45295eb0) at syncobject.hh:31
#2 Dispatcher::run (this=0x7f1f45295eb0) at dispatcher.cc:85
#3 0x00007f1f48df3813 in launch_dispatcher_thread (arg=0x7f1f45295f04) at dispatcher.cc:28
#4 0x00007f1f4d0d79ca in start_thread () from /lib/libpthread.so.0
#5 0x00007f1f4ce356cd in clone () from /lib/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 4 (Thread 16272):
#0 0x00007f1f4cdf935d in nanosleep () from /lib/libc.so.6
#1 0x00007f1f4cdf91d0 in sleep () from /lib/libc.so.6
#2 0x0000000000416618 in check_isasl_db_thread (arg=<value optimized out>) at daemon/isasl.c:233
#3 0x00007f1f4d0d79ca in start_thread () from /lib/libpthread.so.0
#4 0x00007f1f4ce356cd in clone () from /lib/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 3 (Thread 16280):
#0 0x00007f1f4d0dcbc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007f1f48df0d48 in SyncObject::wait (this=0x7f1f46036630, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x7f1f46036630, d=...) at dispatcher.cc:244
#3 0x00007f1f48df2c05 in Dispatcher::run (this=0x7f1f460355a0) at dispatcher.cc:119
#4 0x00007f1f48df3813 in launch_dispatcher_thread (arg=0x7f1f460355f4) at dispatcher.cc:28
#5 0x00007f1f4d0d79ca in start_thread () from /lib/libpthread.so.0
#6 0x00007f1f4ce356cd in clone () from /lib/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 2 (Thread 16277):
#0 0x00007f1f4ce35cc3 in epoll_wait () from /lib/libc.so.6
#1 0x00007f1f4d9a095c in epoll_dispatch (base=0xace500, tv=<value optimized out>) at epoll.c:404
--Type <return> to continue, or q <return> to quit--
#2 0x00007f1f4d98f2cc in event_base_loop (base=0xace500, flags=<value optimized out>) at event.c:1558
#3 0x0000000000414824 in worker_libevent (arg=0xacc020) at daemon/thread.c:304
#4 0x00007f1f4d0d79ca in start_thread () from /lib/libpthread.so.0
#5 0x00007f1f4ce356cd in clone () from /lib/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 1 (Thread 16274):
#0 0x00007f1f4cd82a75 in raise () from /lib/libc.so.6
#1 0x00007f1f4cd865c0 in abort () from /lib/libc.so.6
#2 0x00007f1f4cd7b941 in __assert_fail () from /lib/libc.so.6
#3 0x00007f1f4b945084 in release_handle_locked (h=0x7f1f4bb4a1a0, cookie=0x7f1f3c03c1f8) at bucket_engine.c:597
#4 release_handle (h=0x7f1f4bb4a1a0, cookie=0x7f1f3c03c1f8) at bucket_engine.c:624
#5 get_engine_handle (h=0x7f1f4bb4a1a0, cookie=0x7f1f3c03c1f8) at bucket_engine.c:849
#6 0x00007f1f4b94527b in bucket_tap_notify (handle=0x3f8e, cookie=0x3f92, engine_specific=0x6, nengine=65535,
ttl=16 '\020', tap_flags=25553, tap_event=TAP_OPAQUE, tap_seqno=19015, key=0x7f1f24fb42b4, nkey=0, flags=0,
exptime=0, cas=0, data=0x7f1f24fb42b4, ndata=0, vbucket=0) at bucket_engine.c:1909
#7 0x0000000000409a43 in process_bin_tap_packet (event=TAP_NOOP, c=0x7f1f3c03c1f8) at daemon/memcached.c:2703
#8 0x000000000040efb5 in process_bin_packet (c=0x7f1f3c03c1f8) at daemon/memcached.c:2820
#9 complete_nread_binary (c=0x7f1f3c03c1f8) at daemon/memcached.c:3389
#10 0x000000000040f958 in complete_nread (c=0x7f1f3c03c1f8) at daemon/memcached.c:3471
#11 conn_nread (c=0x7f1f3c03c1f8) at daemon/memcached.c:5275
#12 0x0000000000407263 in event_handler (fd=<value optimized out>, which=<value optimized out>, arg=0x7f1f3c03c1f8)
at daemon/memcached.c:5576
#13 0x00007f1f4d98f3cd in event_process_active_single_queue (base=0xacca20, flags=<value optimized out>) at event.c:1308
#14 event_process_active (base=0xacca20, flags=<value optimized out>) at event.c:1375
#15 event_base_loop (base=0xacca20, flags=<value optimized out>) at event.c:1572
#16 0x0000000000414824 in worker_libevent (arg=0xacbd20) at daemon/thread.c:304
#17 0x00007f1f4d0d79ca in start_thread () from /lib/libpthread.so.0
#18 0x00007f1f4ce356cd in clone () from /lib/libc.so.6
#19 0x0000000000000000 in ?? ()
(gdb)

core file is in 10.1.5.61:/tmp/backup/core.memcached.16270