Description
I was only able to repro this with a deban build, doesn't happen in cluster run. Build was 3.0.0-828
./pyupr -h 10.6.2.118:8091 -o test_open_n_consumer_producers
Program received signal SIGABRT, Aborted.
[Switching to Thread 0x7f2537fb6700 (LWP 4198)]
0x00007f254109d425 in raise () from /lib/x86_64-linux-gnu/libc.so.6
....
Thread 20 (Thread 0x7f25397b9700 (LWP 4201)):
#0 0x00007fff2b5ff98f in clock_gettime ()
#1 0x00007f2541c3c15d in clock_gettime () from /lib/x86_64-linux-gnu/librt.so.1
#2 0x00007f25432ce251 in gethrtime () at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/gethrtime.c:72
#3 0x00007f253cd3e2bc in BlockTimer (o=<optimized out>, n=<optimized out>, d=<optimized out>, this=<optimized out>)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/histo.h:390
#4 EventuallyPersistentStore::flushVBucket (this=0x695d8c0, vbid=<optimized out>)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/ep.cc:2504
#5 0x00007f253cd6fed5 in Flusher::flushVB (this=0x20e3400)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/flusher.cc:281
#6 0x00007f253cd70f3d in Flusher::step (this=0x1, task=0x696f540)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/flusher.cc:172
#7 0x00007f253cd79716 in ExecutorThread::run (this=0x20d2ab0)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/executorthread.cc:94
#8 0x00007f253cd79aad in launch_executor_thread (arg=0x1)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/executorthread.cc:33
#9 0x00007f25432ce09f in platform_thread_wrap (arg=0x20863c0)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#10 0x00007f254204be9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#11 0x00007f254115acbd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#12 0x0000000000000000 in ?? ()
Thread 19 (Thread 0x7f2538fb8700 (LWP 4200)):
#0 0x00007f254115425d in fdatasync () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f253caff8a7 in couch_sync (errinfo=0x208ee20, handle=0x39)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/couchstore/src/os.c:133
#2 0x00007f253cddef86 in cfs_sync(struct
*, couch_file_handle) (errinfo=0x208ee20, h=0x6c17900)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-fs-stats.cc:128
#3 0x00007f253cae8fac in couchstore_commit (db=0x208ee00)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/couchstore/src/couch_db.cc:194
#4 0x00007f253cdd17d0 in CouchKVStore::setVBucketState (this=0x69c8680, vbucketId=<optimized out>, vbstate=...,
vb_change_type=<optimized out>, kvcb=<optimized out>, notify=<optimized out>)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1044
#5 0x00007f253cdd1f29 in CouchKVStore::snapshotVBuckets (this=<optimized out>, vbstates=..., cb=<optimized out>)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:928
#6 0x00007f253cd41770 in EventuallyPersistentStore::snapshotVBuckets (this=0x695d8c0, priority=..., shardId=<optimized out>)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/ep.cc:853
#7 0x00007f253cd9956f in VBSnapshotTask::run (this=<optimized out>)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/tasks.cc:44
#8 0x00007f253cd79716 in ExecutorThread::run (this=0x20d2480)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/executorthread.cc:94
#9 0x00007f253cd79aad in launch_executor_thread (arg=0x39)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/executorthread.cc:33
--Type <return> to continue, or q <return> to quit--
#10 0x00007f25432ce09f in platform_thread_wrap (arg=0x2086460)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#11 0x00007f254204be9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#12 0x00007f254115acbd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#13 0x0000000000000000 in ?? ()
Thread 18 (Thread 0x7f25387b7700 (LWP 4199)):
#0 0x00007f253cd31fc8 in EventuallyPersistentStore::visit (this=<optimized out>, visitor=...)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/ep.cc:2770
#1 0x00007f253cd41725 in EventuallyPersistentStore::snapshotVBuckets (this=0x695d8c0, priority=..., shardId=<optimized out>)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/ep.cc:849
#2 0x00007f253cd9956f in VBSnapshotTask::run (this=<optimized out>)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/tasks.cc:44
#3 0x00007f253cd79716 in ExecutorThread::run (this=0x20d23f0)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/executorthread.cc:94
#4 0x00007f253cd79aad in launch_executor_thread (arg=0x695d8e8)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/executorthread.cc:33
#5 0x00007f25432ce09f in platform_thread_wrap (arg=0x2086450)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#6 0x00007f254204be9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f254115acbd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()
Thread 17 (Thread 0x7f2537fb6700 (LWP 4198)):
#0 0x00007f254109d425 in raise () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f25410a0b8b in abort () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00007f253cda18ed in Processer::run (this=0x20d2d80)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/upr-consumer.cc:59
#3 0x00007f253cd79716 in ExecutorThread::run (this=0x20d2360)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/executorthread.cc:94
#4 0x00007f253cd79aad in launch_executor_thread (arg=0xf8b)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/executorthread.cc:33
#5 0x00007f25432ce09f in platform_thread_wrap (arg=0x2086440)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#6 0x00007f254204be9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f254115acbd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()
Thread 13 (Thread 0x7f2540c4d700 (LWP 3980)):
#0 0x00007f254114d8bd in read () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f25410e1ff8 in _IO_file_underflow () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00007f25410e303e in _IO_default_uflow () from /lib/x86_64-linux-gnu/libc.so.6
#3 0x00007f25410d718a in _IO_getline_info () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x00007f25410d606b in fgets () from /lib/x86_64-linux-gnu/libc.so.6
--Type <return> to continue, or q <return> to quit--
#5 0x00007f2540c4ea91 in fgets (__stream=<optimized out>, __n=<optimized out>, __s=<optimized out>) at /usr/include/bits/stdio2.h:255
#6 check_stdin_thread (arg=<optimized out>)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/memcached/extensions/daemon/stdin_check.c:38
#7 0x00007f25432ce09f in platform_thread_wrap (arg=0x2086060)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#8 0x00007f254204be9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#9 0x00007f254115acbd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#10 0x0000000000000000 in ?? ()
Thread 12 (Thread 0x7f2540248700 (LWP 3981)):
#0 0x00007f25420500fe in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f25432ce18b in cb_cond_timedwait (cond=0x7f254044c240, mutex=0x7f254044c200, ms=<optimized out>)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/cb_pthreads.c:156
#2 0x00007f254024b1e8 in logger_thead_main (arg=<optimized out>)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/memcached/extensions/loggers/file_logger.c:372
#3 0x00007f25432ce09f in platform_thread_wrap (arg=0x2086070)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x00007f254204be9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007f254115acbd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()
Thread 11 (Thread 0x7f253f83a700 (LWP 3982)):
#0 0x00007f254115b353 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f25428635a6 in epoll_dispatch (base=0x693a280, tv=<optimized out>) at epoll.c:404
#2 0x00007f254284ea04 in event_base_loop (base=0x693a280, flags=<optimized out>) at event.c:1558
#3 0x00007f25432ce09f in platform_thread_wrap (arg=0x20860f0)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x00007f254204be9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007f254115acbd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()
Thread 10 (Thread 0x7f253f039700 (LWP 3983)):
#0 0x00007f254115b353 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f25428635a6 in epoll_dispatch (base=0x693a500, tv=<optimized out>) at epoll.c:404
#2 0x00007f254284ea04 in event_base_loop (base=0x693a500, flags=<optimized out>) at event.c:1558
#3 0x00007f25432ce09f in platform_thread_wrap (arg=0x2086100)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x00007f254204be9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007f254115acbd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()
Thread 9 (Thread 0x7f253e838700 (LWP 3984)):
#0 0x00007f254115b353 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f25428635a6 in epoll_dispatch (base=0x693a780, tv=<optimized out>) at epoll.c:404
--Type <return> to continue, or q <return> to quit--
#2 0x00007f254284ea04 in event_base_loop (base=0x693a780, flags=<optimized out>) at event.c:1558
#3 0x00007f25432ce09f in platform_thread_wrap (arg=0x2086110)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x00007f254204be9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007f254115acbd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()
Thread 8 (Thread 0x7f253e037700 (LWP 3985)):
#0 0x00007f254115b353 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f25428635a6 in epoll_dispatch (base=0x693aa00, tv=<optimized out>) at epoll.c:404
#2 0x00007f254284ea04 in event_base_loop (base=0x693aa00, flags=<optimized out>) at event.c:1558
#3 0x00007f25432ce09f in platform_thread_wrap (arg=0x2086120)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x00007f254204be9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007f254115acbd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()
Thread 7 (Thread 0x7f253d836700 (LWP 3986)):
#0 0x00007f254115b353 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f25428635a6 in epoll_dispatch (base=0x693ac80, tv=<optimized out>) at epoll.c:404
#2 0x00007f254284ea04 in event_base_loop (base=0x693ac80, flags=<optimized out>) at event.c:1558
#3 0x00007f25432ce09f in platform_thread_wrap (arg=0x2086130)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x00007f254204be9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007f254115acbd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()
Thread 6 (Thread 0x7f253a7bb700 (LWP 3987)):
#0 0x00007f254112683d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f2541154774 in usleep () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00007f253cd77c35 in updateStatsThread (arg=<optimized out>)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/ep-engine/src/memory_tracker.cc:36
#3 0x00007f25432ce09f in platform_thread_wrap (arg=0x20861f0)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/platform/src/cb_pthreads.c:19
#4 0x00007f254204be9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007f254115acbd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()
Thread 1 (Thread 0x7f2543aea740 (LWP 3979)):
#0 0x00007f254115b353 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f25428635a6 in epoll_dispatch (base=0x693a000, tv=<optimized out>) at epoll.c:404
#2 0x00007f254284ea04 in event_base_loop (base=0x693a000, flags=<optimized out>) at event.c:1558
#3 0x00000000004103e3 in main (argc=<optimized out>, argv=<optimized out>)
at /home/buildbot/ubuntu-1004-x64-300-builder/build/build/memcached/daemon/memcached.c:8794