Erlang crash when obtain diags / IncompleteRead: erl_exit (n=1, fmt=0x591318 "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n") at beam/erl_init.c:1613

build 2.0.1-139-rel
all tests failed when connect to node 10.3.121.93
http://qa.hq.northscale.net/job/centos-64-2.0-rebalance-regressions/169/consoleFull

First, I am not sure that core.beam.smp.29903 was obtained during the tests, as creation time is different for 2 days. although 2 days ago tests were not performed, and the previous launches were successful.

root@ubuntu1104-64:/tmp/backup# ls -la
total 5234920
drwxr-xr-x 2 root root 4096 2013-01-19 00:32 .
drwxrwxrwt 3 root root 4096 2013-01-21 03:17 ..
rw------ 1 couchbase couchbase 2726223872 2013-01-19 00:01 core.beam.smp.29903
rw------ 1 couchbase couchbase 1241178112 2012-12-06 14:04 core.memcached.28835
rw------ 1 couchbase couchbase 1570488320 2012-09-21 06:45 core.memcached.4751
rw------ 1 couchbase couchbase 454139904 2012-10-18 00:24 core.memcached.8944
root@ubuntu1104-64:/tmp/backup# locate erl_crash.dump
/opt/couchbase/var/lib/couchbase/erl_crash.dump.01-18-2013-05:02:25.29876
root@ubuntu1104-64:/tmp/backup# cd /opt/couchbase/var/lib/couchbase
root@ubuntu1104-64:/opt/couchbase/var/lib/couchbase# ls -la
total 48
drwxr-xr-x 7 couchbase couchbase 4096 2013-01-21 04:15 .
drwxr-xr-x 5 couchbase couchbase 4096 2013-01-21 01:28 ..
drwxr-xr-x 2 couchbase couchbase 4096 2013-01-21 02:00 config
rw-rr- 1 couchbase couchbase 17 2013-01-21 03:48 couchbase-server.cookie
rw-rr- 1 couchbase couchbase 17 2013-01-21 03:48 couchbase-server.node
rw-rr- 1 couchbase root 5 2013-01-21 03:48 couchbase-server.pid
drwxr-xr-x 3 couchbase couchbase 4096 2013-01-21 03:48 data
rw-rr- 1 couchbase couchbase 11 2013-01-21 01:28 ip
rw-rr- 1 couchbase couchbase 0 2013-01-18 19:00 ip_start
drwxr-xr-x 2 couchbase couchbase 4096 2013-01-21 03:48 logs
drwxr-xr-x 2 couchbase couchbase 4096 2013-01-21 04:15 mnesia
rw-rr- 1 couchbase couchbase 344 2013-01-21 04:15 remote_clusters_cache
drwxr-xr-x 2 couchbase couchbase 4096 2013-01-21 01:28 tmp
root@ubuntu1104-64:/opt/couchbase/var/lib/couchbase# updatedb
root@ubuntu1104-64:/opt/couchbase/var/lib/couchbase# locate erl_crash.dump
root@ubuntu1104-64:/opt/couchbase/var/lib/couchbase#

Secondly it seems that this is аn erlang problem

root@ubuntu1104-64:/tmp/backup# sudo gdb /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp core.beam.smp.29903
GNU gdb (Ubuntu/Linaro 7.2-1ubuntu11) 7.2
Copyright (C) 2010 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp...done.
[New Thread 29926]
[New Thread 29939]
[New Thread 29927]
[New Thread 29906]
[New Thread 29903]
[New Thread 29914]
[New Thread 29909]
[New Thread 29907]
[New Thread 29918]
[New Thread 29911]
[New Thread 29905]
[New Thread 29908]
[New Thread 29921]
[New Thread 29912]
[New Thread 29917]
[New Thread 29910]
[New Thread 29923]
[New Thread 29922]
[New Thread 29915]
[New Thread 29919]
[New Thread 29913]
[New Thread 29924]
[New Thread 29920]
[New Thread 29928]
[New Thread 29925]
[New Thread 29916]

warning: Can't read pathname for load map: Input/output error.
Reading symbols from /lib/x86_64-linux-gnu/libutil.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libutil.so.1
Reading symbols from /lib/x86_64-linux-gnu/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libdl.so.2
Reading symbols from /lib/x86_64-linux-gnu/libm.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libm.so.6
Reading symbols from /lib/libncurses.so.5...(no debugging symbols found)...done.
Loaded symbols for /lib/libncurses.so.5
Reading symbols from /lib/x86_64-linux-gnu/libpthread.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libpthread.so.0
Reading symbols from /lib/x86_64-linux-gnu/librt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/librt.so.1
Reading symbols from /lib/x86_64-linux-gnu/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /opt/couchbase/lib/ns_server/erlang/lib/mlockall/priv/mlockall_nif.so...done.
Loaded symbols for /opt/couchbase/lib/ns_server/erlang/lib/mlockall/priv/mlockall_nif.so
Reading symbols from /opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so...done.
Loaded symbols for /opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so
Reading symbols from /usr/lib64/libcrypto.so.0.9.8...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libcrypto.so.0.9.8
Reading symbols from /usr/lib64/libssl.so.0.9.8...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libssl.so.0.9.8
Reading symbols from /lib/x86_64-linux-gnu/libz.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libz.so.1
Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-f3bff70-git/priv/lib/couch_icu_driver.so...done.
Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-f3bff70-git/priv/lib/couch_icu_driver.so
Reading symbols from /opt/couchbase/lib/libv8.so...done.
Loaded symbols for /opt/couchbase/lib/libv8.so
Reading symbols from /opt/couchbase/lib/libicuuc.so.44...done.
Loaded symbols for /opt/couchbase/lib/libicuuc.so.44
Reading symbols from /opt/couchbase/lib/libicudata.so.44...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/lib/libicudata.so.44
Reading symbols from /opt/couchbase/lib/libicui18n.so.44...done.
Loaded symbols for /opt/couchbase/lib/libicui18n.so.44
Reading symbols from /lib/x86_64-linux-gnu/libcrypt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libcrypt.so.1
Reading symbols from /usr/lib/x86_64-linux-gnu/libstdc++.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libstdc++.so.6
Reading symbols from /lib/x86_64-linux-gnu/libgcc_s.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libgcc_s.so.1
Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so...done.
Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so
Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so...done.
Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so
Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so...done.
Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so
Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-f3bff70-git/priv/lib/couch_ejson_compare.so...done.
Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-f3bff70-git/priv/lib/couch_ejson_compare.so
Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/couch_view_parser-1.0/priv/couch_view_parser_nif.so...done.
Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/couch_view_parser-1.0/priv/couch_view_parser_nif.so
Core was generated by `/opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp -A 16 -sbt u -P 327680 -K tru'.
Program terminated with signal 6, Aborted.
#0 0x00007f13ec2ddd05 in raise () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) t a a bt

Thread 26 (Thread 29916):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead15434, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 25 (Thread 29925):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x86ced4, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x0000000000475a4f in erts_cnd_wait (prepare=<value optimized out>, resume=0x4916e0 <resume_after_block>, arg=0x7f13eb8273c0) at beam/erl_threads.h:697
#3 erts_smp_cnd_wait (prepare=<value optimized out>, resume=0x4916e0 <resume_after_block>, arg=0x7f13eb8273c0) at beam/erl_smp.h:472
#4 block_me (prepare=<value optimized out>, resume=0x4916e0 <resume_after_block>, arg=0x7f13eb8273c0) at beam/utils.c:3439
#5 erts_block_me (prepare=<value optimized out>, resume=0x4916e0 <resume_after_block>, arg=0x7f13eb8273c0) at beam/utils.c:3487
#6 0x00000000004a0312 in erts_smp_chk_system_block (p=<value optimized out>, calls=<value optimized out>) at beam/sys.h:894
#7 schedule (p=<value optimized out>, calls=<value optimized out>) at beam/erl_process.c:5373
#8 0x0000000000523b3f in process_main () at beam/beam_emu.c:1225
#9 0x00000000004938b6 in sched_thread_func (vesdp=<value optimized out>) at beam/erl_process.c:3782
#10 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#11 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#12 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#13 0x0000000000000000 in ?? ()

Thread 24 (Thread 29928):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x000000000058c483 in wait__ (e=0x22a5d20) at pthread/ethr_event.c:185
#2 ethr_event_wait (e=0x22a5d20) at pthread/ethr_event.c:218
#3 0x000000000049e70d in erts_tse_wait (fcalls=<value optimized out>, esdp=0x7f13eb852200, rq=0x7f13eb8279c0) at beam/erl_threads.h:1809
#4 scheduler_wait (fcalls=<value optimized out>, esdp=0x7f13eb852200, rq=0x7f13eb8279c0) at beam/erl_process.c:1076
#5 0x00000000004a0788 in schedule (p=<value optimized out>, calls=<value optimized out>) at beam/erl_process.c:5421
#6 0x0000000000523b3f in process_main () at beam/beam_emu.c:1225
#7 0x00000000004938b6 in sched_thread_func (vesdp=<value optimized out>) at beam/erl_process.c:3782
#8 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#9 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#11 0x0000000000000000 in ?? ()

Thread 23 (Thread 29920):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead15614, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 22 (Thread 29924):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
--Type <return> to continue, or q <return> to quit--
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x8720c4, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x000000000053f31f in erts_cnd_wait (unused=<value optimized out>) at beam/erl_threads.h:697
#3 child_waiter (unused=<value optimized out>) at sys/unix/sys.c:2835
#4 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#5 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 21 (Thread 29913):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead152cc, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 20 (Thread 29919):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead1559c, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 19 (Thread 29915):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead153bc, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 18 (Thread 29922):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead15704, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 17 (Thread 29923):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead1577c, mtx=0x80) at common/ethr_mutex.c:1350
--Type <return> to continue, or q <return> to quit--
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 16 (Thread 29910):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead15164, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0

#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 15 (Thread 29917):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead154ac, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 14 (Thread 29912):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead15254, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 13 (Thread 29921):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead1568c, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 12 (Thread 29908):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead15074, mtx=0x80) at common/ethr_mutex.c:1350
--Type <return> to continue, or q <return> to quit--
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 11 (Thread 29905):
#0 0x00007f13ec858cbd in read () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x000000000053e250 in read (unused=<value optimized out>) at /usr/include/bits/unistd.h:45
#2 signal_dispatcher_thread_func (unused=<value optimized out>) at sys/unix/sys.c:2906
#3 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#4 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 10 (Thread 29911):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead151dc, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 9 (Thread 29918):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead15524, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 8 (Thread 29907):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x827204, mtx=0x80) at common/ethr_mutex.c:1350

#2 0x0000000000468679 in erts_cnd_wait (unused=<value optimized out>) at beam/erl_threads.h:697
#3 erts_smp_cnd_wait (unused=<value optimized out>) at beam/erl_smp.h:472
#4 sys_msg_dispatcher_func (unused=<value optimized out>) at beam/erl_trace.c:3238
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 7 (Thread 29909):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead150ec, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
--Type <return> to continue, or q <return> to quit--
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 6 (Thread 29914):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x0000000000589da9 in ethr_cond_wait (cnd=0x7f13ead15344, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ee48b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 5 (Thread 29903):
#0 0x00007f13ec38b0d3 in select () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x000000000053dc70 in erts_sys_main_thread () at sys/unix/sys.c:3049
#2 0x0000000000449ab1 in erl_start (argc=57, argv=<value optimized out>) at beam/erl_init.c:1441
#3 0x000000000042b289 in main (argc=0, argv=0x0) at sys/unix/erl_main.c:29

Thread 4 (Thread 29906):
#0 0x00007f13ec38b0d3 in select () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x000000000058fb21 in erts_milli_sleep (ms=<value optimized out>) at common/erl_misc_utils.c:159
#2 0x000000000046fff9 in emergency_watchdog (unused=<value optimized out>) at beam/utils.c:3959
#3 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#4 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 3 (Thread 29927):
#0 0x00007f13ec855bac in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x000000000058c483 in wait__ (e=0x22a5c60) at pthread/ethr_event.c:185
#2 ethr_event_wait (e=0x22a5c60) at pthread/ethr_event.c:218
#3 0x000000000049e70d in erts_tse_wait (fcalls=<value optimized out>, esdp=0x7f13eb844140, rq=0x7f13eb8277c0) at beam/erl_threads.h:1809
#4 scheduler_wait (fcalls=<value optimized out>, esdp=0x7f13eb844140, rq=0x7f13eb8277c0) at beam/erl_process.c:1076
#5 0x00000000004a0788 in schedule (p=<value optimized out>, calls=<value optimized out>) at beam/erl_process.c:5421
#6 0x0000000000523b3f in process_main () at beam/beam_emu.c:1225
#7 0x00000000004938b6 in sched_thread_func (vesdp=<value optimized out>) at beam/erl_process.c:3782
#8 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#9 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#11 0x0000000000000000 in ?? ()

Thread 2 (Thread 29939):
#0 0x00007f13ec8594bd in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f13e25946da in terminatorLoop (args=<value optimized out>) at mapreduce_nif.cc:450
#2 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#3 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 1 (Thread 29926):
--Type <return> to continue, or q <return> to quit--
#0 0x00007f13ec2ddd05 in raise () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f13ec2e1ab6 in abort () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00000000004489cd in erl_exit (n=1, fmt=0x591318 "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n") at beam/erl_init.c:1613
#3 0x000000000042decc in erts_alc_fatal_error (error=1, func=0, n=130) at beam/erl_alloc.c:1642
#4 0x000000000042df87 in erts_alloc_n_enomem (n=6, size=18446744073709551615) at beam/erl_alloc.c:1667
#5 0x00000000004f097b in erts_alloc (p=0x7f13dd9113c0, new_sz=<value optimized out>, objv=<value optimized out>, nobj=<value optimized out>) at beam/erl_alloc.h:205
#6 do_minor (p=0x7f13dd9113c0, new_sz=<value optimized out>, objv=<value optimized out>, nobj=<value optimized out>) at beam/erl_gc.c:897
#7 0x00000000004f4815 in minor_collection (p=0x7f13dd9113c0, objv=0x7f13e9beccd0, nobj=1, need=<value optimized out>) at beam/erl_gc.c:769
#8 erts_garbage_collect (p=0x7f13dd9113c0, objv=0x7f13e9beccd0, nobj=1, need=<value optimized out>) at beam/erl_gc.c:376
#9 0x00000000004f4d19 in erts_gc_after_bif_call (p=0x7f13dd9113c0, result=139719498648234, regs=<value optimized out>, arity=18446744073709551615) at beam/erl_gc.c:326
#10 0x0000000000526ce8 in process_main () at beam/beam_emu.c:2508
#11 0x00000000004938b6 in sched_thread_func (vesdp=<value optimized out>) at beam/erl_process.c:3782
#12 0x000000000058bd89 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#13 0x00007f13ec850d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#14 0x00007f13ec392fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#15 0x0000000000000000 in ?? ()