Details
Description
This is happening possibly because the user is sending data to a moxi during rebalance when the node is being added or removed
Thread 8 (Thread 17007):
#0 0x0000000000433373 in cJSON_GetArrayItem (array=0x26db250, item=244)
at cJSON.c:451
#1 0x0000000000482b70 in populate_buckets (vb=0x2773bb0, c=0x26db250,
is_ft=<value optimized out>) at src/vbucket.c:175
#2 0x0000000000483108 in parse_cjson (c=<value optimized out>)
at src/vbucket.c:273
#3 0x0000000000482d6d in parse_cjson (c=0x1fbda30) at src/vbucket.c:205
#4 0x00000000004831d0 in vbucket_config_parse_string (
data=<value optimized out>) at src/vbucket.c:295
#5 0x0000000000432306 in lvb_create (ptr=0x25adcf0,
config=0x276f470 "{\n\t\"name\":\t\"default\",\n\t\"nodeLocator\":\t\"vbucket\",\n\t\"saslPassword\":\t\"\",\n\t\"nodes\":\t[{\n\t\t\t\"replication\":\t0,\n\t\t\t\"clusterMembership\":\t\"active\",\n\t\t\t\"status\":\t\"healthy\",\n\t\t\t\"hostname\":\t\"10.1.5.38:8091\",\n\t\t\t\"c"...,
default_usr=0x1d4d46c "default", default_pwd=0x0,
opts=0x30313231313a33 <Address 0x30313231313a33 out of bounds>)
at mcs.c:157
#6 0x000000000041a35b in init_mcs_st (mst=0x26db250,
config=0xf4 <Address 0xf4 out of bounds>,
default_usr=0x2 <Address 0x2 out of bounds>,
default_pwd=0x1 <Address 0x1 out of bounds>,
opts=0x30313231313a33 <Address 0x30313231313a33 out of bounds>)
at cproxy.c:1320
#7 0x000000000041a4f8 in cproxy_create_downstream (
config=<value optimized out>, config_ver=161, behavior_pool=0x1d4cee8)
at cproxy.c:1293
#8 0x000000000041cfa9 in cproxy_add_downstream (ptd=0x1d4ced0)
at cproxy.c:882
#9 0x000000000041e9d6 in cproxy_reserve_downstream (ptd=0x1d4ced0)
at cproxy.c:908
#10 0x000000000041ecfe in cproxy_assign_downstream (ptd=0x1d4ced0)
at cproxy.c:1783
#11 0x000000000042aea9 in cproxy_process_upstream_binary_nread (
c=0x7f3b1c02a060) at cproxy_protocol_b.c:188
#12 0x000000000040fb7b in complete_nread (c=0x7f3b1c02a060)
at memcached.c:1990
#13 drive_machine (c=0x7f3b1c02a060) at memcached.c:3576
#14 0x0000000000446c69 in event_process_active_single_queue (base=0x1c912d0,
flags=<value optimized out>) at event.c:1308
#15 event_process_active (base=0x1c912d0, flags=<value optimized out>)
at event.c:1375
#16 event_base_loop (base=0x1c912d0, flags=<value optimized out>)
at event.c:1572
#17 0x0000000000416125 in worker_libevent (arg=<value optimized out>)
at thread.c:272
#18 0x00007f3b271a69ca in start_thread () from /lib/libpthread.so.0
#19 0x00007f3b26f046cd in clone () from /lib/libc.so.6
#20 0x0000000000000000 in ?? ()
Thread 7 (Thread 17005):
#0 0x00007f3b271ae99d in close () from /lib/libpthread.so.0
#1 0x0000000000409fdb in conn_close (c=0x1f63d30) at memcached.c:559
#2 0x000000000040f2ba in drive_machine (c=0x1f63d30) at memcached.c:3761
#3 0x0000000000446c69 in event_process_active_single_queue (base=0x1c885f0,
flags=<value optimized out>) at event.c:1308
#4 event_process_active (base=0x1c885f0, flags=<value optimized out>)
at event.c:1375
#5 event_base_loop (base=0x1c885f0, flags=<value optimized out>)
at event.c:1572
#6 0x0000000000416125 in worker_libevent (arg=<value optimized out>)
at thread.c:272
#7 0x00007f3b271a69ca in start_thread () from /lib/libpthread.so.0
#8 0x00007f3b26f046cd in clone () from /lib/libc.so.6
#9 0x0000000000000000 in ?? ()
Thread 6 (Thread 17006):
#0 0x00007f3b26f04c6a in epoll_ctl () from /lib/libc.so.6
#1 0x000000000044c889 in epoll_apply_one_change (base=<value optimized out>,
epollop=0x1c8ce60, ch=0x7f3b25e1a8a0) at epoll.c:262
#2 0x000000000044cd8c in epoll_nochangelist_add (base=0x1d, fd=203,
old=<value optimized out>, events=-1, p=<value optimized out>)
at epoll.c:363
#3 0x0000000000448a8a in evmap_io_add (base=0x1c8cbe0,
fd=<value optimized out>, ev=0x1df3030) at evmap.c:318
#4 0x000000000044509e in event_add_internal (ev=0x1df3030, tv=0x0,
tv_is_absolute=0) at event.c:2016
#5 0x00000000004452e6 in event_add (ev=0x1df3030, tv=0x0) at event.c:1909
#6 0x0000000000409d97 in update_event_timed_real (c=0x1df3020,
new_flags=<value optimized out>, update_diag=<value optimized out>)
at memcached.c:3339
#7 update_event_real (c=0x1df3020, new_flags=<value optimized out>,
update_diag=<value optimized out>) at memcached.c:3320
#8 0x000000000042c181 in b2b_forward_item_vbucket (uc=0x1e6d2f0,
d=0x7f3b141ac750, it=0x7f3b14377350, c=0x1df3020, vbucket=906)
at cproxy_protocol_b2b.c:226
#9 0x000000000042c7a3 in b2b_forward_item (uc=0x1e6d2f0, d=0x7f3b141ac750,
it=0x7f3b14377350) at cproxy_protocol_b2b.c:176
#10 0x000000000041e776 in cproxy_forward_or_error (dc=<value optimized out>,
closing=false) at cproxy.c:1959
#11 zstored_release_downstream_conn (dc=<value optimized out>, closing=false)
at cproxy.c:3436
#12 0x000000000041cb95 in cproxy_release_downstream (d=0x7f3b140129f0,
force=false) at cproxy.c:1167
#13 0x000000000041fc02 in cproxy_release_downstream_conn (c=0x1df3020)
at cproxy.c:2140
#14 cproxy_on_pause_downstream_conn (c=0x1df3020) at cproxy.c:2162
#15 0x000000000040f6db in drive_machine (c=0x1df3020) at memcached.c:3749
#16 0x0000000000446c69 in event_process_active_single_queue (base=0x1c8cbe0,
flags=<value optimized out>) at event.c:1308
#17 event_process_active (base=0x1c8cbe0, flags=<value optimized out>)
at event.c:1375
#18 event_base_loop (base=0x1c8cbe0, flags=<value optimized out>)
at event.c:1572
#19 0x0000000000416125 in worker_libevent (arg=<value optimized out>)
at thread.c:272
#20 0x00007f3b271a69ca in start_thread () from /lib/libpthread.so.0
#21 0x00007f3b26f046cd in clone () from /lib/libc.so.6
#22 0x0000000000000000 in ?? ()
Thread 5 (Thread 17003):
#0 0x00007f3b26f04cc3 in epoll_wait () from /lib/libc.so.6
#1 0x000000000044cba8 in epoll_dispatch (base=0x1c711c0,
tv=<value optimized out>) at epoll.c:404
#2 0x0000000000446b75 in event_base_loop (base=0x1c711c0,
flags=<value optimized out>) at event.c:1558
#3 0x00000000004091a9 in main (argc=12, argv=0x7fffda3317b8)
at memcached.c:5061
Thread 4 (Thread 17010):
#0 0x00007f3b26ef647d in read () from /lib/libc.so.6
#1 0x00007f3b26e91348 in _IO_file_underflow () from /lib/libc.so.6
#2 0x00007f3b26e92eee in _IO_default_uflow () from /lib/libc.so.6
#3 0x00007f3b26e89c7b in getc () from /lib/libc.so.6
#4 0x0000000000432ba6 in check_stdin_thread (arg=<value optimized out>)
at stdin_check.c:18
#5 0x00007f3b271a69ca in start_thread () from /lib/libpthread.so.0
#6 0x00007f3b26f046cd in clone () from /lib/libc.so.6
#7 0x0000000000000000 in ?? ()
Thread 3 (Thread 17009):
#0 0x00007f3b26ef7f53 in poll () from /lib/libc.so.6
#1 0x00000000004556d6 in Curl_socket_ready ()
#2 0x000000000046ca5d in Curl_do_perform ()
#3 0x000000000044ee6e in run_rest_conflate (arg=0x1c9d440) at rest.c:293
#4 0x00007f3b271a69ca in start_thread () from /lib/libpthread.so.0
#5 0x00007f3b26f046cd in clone () from /lib/libc.so.6
#6 0x0000000000000000 in ?? ()
Thread 2 (Thread 17008):
#0 0x00007f3b271ab85c in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/libpthread.so.0
#1 0x000000000041527e in assoc_maintenance_thread (arg=<value optimized out>)
at assoc.c:220
#2 0x00007f3b271a69ca in start_thread () from /lib/libpthread.so.0
#3 0x00007f3b26f046cd in clone () from /lib/libc.so.6
#4 0x0000000000000000 in ?? ()
Thread 1 (Thread 17004):
#0 0x0000000000000000 in ?? ()
#1 0x0000000000000000 in ?? ()