erlang core dump in test rebalarebalance_in_with_ddoc_compaction with rebalance in 1 node and compaction ( ubuntu 10_64)

version=2.0.0-1846-rel
http://qa.hq.northscale.net/job/ubuntu-64-2.0-basic-new-rebalance-tests/206/consoleFull
./testrunner -i /tmp/rebalance-tests.ini get-logs=True,GROUP=BASIC -t rebalance.rebalancein.RebalanceInTests.rebalarebalance_in_with_ddoc_compaction,items=500000,max_verify=100000,GROUP=IN;BASIC;COMPACTION

DISTRIB_DESCRIPTION="Ubuntu 10.04.3 LTS"

crash on node 10.1.3.119

[2012-10-13 20:49:30,032] - [task:1537] INFO - stats compaction before triggering it: (0,86.489398529)
[2012-10-13 20:49:30,040] - [rest_client:1395] INFO - compaction for ddoc '_design%2fddoc1' was triggered
[2012-10-13 20:49:30,041] - [task:209] INFO - adding node 10.1.3.111:8091 to cluster
[2012-10-13 20:49:30,042] - [rest_client:761] INFO - adding remote node : 10.1.3.111 to this cluster @ : 10.1.3.119
[2012-10-13 20:49:31,091] - [rest_client:858] INFO - rebalance params : password=password&ejectedNodes=&user=Administrator&knownNodes=ns_1%4010.1.3.119%2Cns_1%4010.1.3.111
[2012-10-13 20:49:31,098] - [rest_client:865] INFO - rebalance operation started
[2012-10-13 20:49:31,141] - [rest_client:961] INFO - rebalance percentage : 0 %
[2012-10-13 20:49:32,335] - [task:1556] INFO - stats compaction: (0,86.4896851075)
[2012-10-13 20:49:32,393] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:49:35,526] - [task:1556] INFO - stats compaction: (0,86.4628249758)
[2012-10-13 20:49:35,572] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:49:38,727] - [task:1556] INFO - stats compaction: (0,86.3709725875)
[2012-10-13 20:49:38,917] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:49:41,923] - [rest_client:961] INFO - rebalance percentage : 4.16666666667 %
[2012-10-13 20:49:42,100] - [task:1556] INFO - stats compaction: (0,86.3867911234)
[2012-10-13 20:49:42,407] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:49:45,660] - [task:1556] INFO - stats compaction: (0,86.4228376914)
[2012-10-13 20:49:45,761] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:49:48,184] - [task:1556] INFO - stats compaction: (0,86.4620036959)
[2012-10-13 20:49:53,357] - [rest_client:961] INFO - rebalance percentage : 8.33333333333 %
[2012-10-13 20:49:55,713] - [task:1556] INFO - stats compaction: (0,86.6259215817)
[2012-10-13 20:49:55,775] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:49:58,966] - [task:1556] INFO - stats compaction: (0,86.7158384029)
[2012-10-13 20:49:59,079] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:02,295] - [task:1556] INFO - stats compaction: (0,86.750024191)
[2012-10-13 20:50:02,346] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:04,352] - [rest_client:961] INFO - rebalance percentage : 12.5 %
[2012-10-13 20:50:04,506] - [task:1556] INFO - stats compaction: (0,86.780168592)
[2012-10-13 20:50:04,588] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:07,957] - [task:1556] INFO - stats compaction: (0,86.85154036)
[2012-10-13 20:50:08,027] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:11,324] - [task:1556] INFO - stats compaction: (0,86.9334326418)
[2012-10-13 20:50:11,622] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:13,761] - [task:1556] INFO - stats compaction: (0,87.038179628)
[2012-10-13 20:50:13,826] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:14,841] - [rest_client:961] INFO - rebalance percentage : 15.625 %
[2012-10-13 20:50:16,038] - [task:1556] INFO - stats compaction: (0,87.0361382468)
[2012-10-13 20:50:16,091] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:19,474] - [task:1556] INFO - stats compaction: (0,87.0992259371)
[2012-10-13 20:50:19,527] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:22,658] - [task:1556] INFO - stats compaction: (0,87.1670086428)
[2012-10-13 20:50:22,786] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:25,795] - [rest_client:961] INFO - rebalance percentage : 17.7083333333 %
[2012-10-13 20:50:25,960] - [task:1556] INFO - stats compaction: (0,87.2735580996)
[2012-10-13 20:50:26,020] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:28,289] - [task:1556] INFO - stats compaction: (0,87.2720171812)
[2012-10-13 20:50:28,337] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:30,487] - [task:1556] INFO - stats compaction: (0,87.3678748438)
[2012-10-13 20:50:30,710] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:32,871] - [task:1556] INFO - stats compaction: (0,87.3537544788)
[2012-10-13 20:50:32,935] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:35,171] - [task:1556] INFO - stats compaction: (0,87.3979536858)
[2012-10-13 20:50:35,378] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:50:36,388] - [rest_client:961] INFO - rebalance percentage : 20.8333333333 %
[2012-10-13 20:50:37,678] - [task:1556] INFO - stats compaction: (0,87.4671980941)

[2012-10-13 20:54:57,007] - [rest_client:961] INFO - rebalance percentage : 71.6145833333 %
[2012-10-13 20:54:58,632] - [task:1556] INFO - stats compaction: (0,91.0931966289)
[2012-10-13 20:54:58,887] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:55:02,542] - [task:1556] INFO - stats compaction: (0,91.0931966289)
[2012-10-13 20:55:02,748] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:55:05,310] - [task:1556] INFO - stats compaction: (0,91.0931966289)
[2012-10-13 20:55:05,507] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:55:07,520] - [rest_client:961] INFO - rebalance percentage : 81.5104166667 %
[2012-10-13 20:55:08,091] - [task:1556] INFO - stats compaction: (0,91.0931966289)
[2012-10-13 20:55:08,312] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:55:10,702] - [task:1556] INFO - stats compaction: (0,91.0931966289)
[2012-10-13 20:55:10,948] - [task:1561] INFO - design doc ddoc1 is compacting
[2012-10-13 20:55:14,126] - [task:1556] INFO - stats compaction: (1,90.3651030309)
[2012-10-13 20:55:14,126] - [task:1564] INFO - compactor was run, compaction count was changed on 1
[2012-10-13 20:55:14,127] - [task:1567] INFO - fragmentation went from 86 to 90
[2012-10-13 20:55:14,311] - [task:1576] INFO - compaction was completed, but fragmentation value 90.3651030309 is more than before compaction 86.489398529
COMPACTION FAIL

gdb /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp /tmp/core.beam.smp.20262
GNU gdb (GDB) 7.1-ubuntu
Copyright (C) 2010 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp...done.
[New Thread 20291]
[New Thread 25691]
[New Thread 20271]
[New Thread 25690]
[New Thread 20273]
[New Thread 20262]
[New Thread 20277]
[New Thread 20279]
[New Thread 20269]
[New Thread 20283]
[New Thread 20288]
[New Thread 20284]
[New Thread 20268]
[New Thread 20290]
[New Thread 20293]
[New Thread 20274]
[New Thread 20282]
[New Thread 20286]
[New Thread 20302]
[New Thread 20275]
[New Thread 20276]
[New Thread 20289]
[New Thread 20278]
[New Thread 20281]
[New Thread 20280]
[New Thread 20285]
[New Thread 20287]
[New Thread 20292]

warning: Can't read pathname for load map: Input/output error.
Reading symbols from /lib/libutil.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/libutil.so.1
Reading symbols from /lib/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib/libdl.so.2
Reading symbols from /lib/libm.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib/libm.so.6
Reading symbols from /lib/libncurses.so.5...(no debugging symbols found)...done.
Loaded symbols for /lib/libncurses.so.5
Reading symbols from /lib/libpthread.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib/libpthread.so.0
Reading symbols from /lib/librt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/librt.so.1
Reading symbols from /lib/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so...done.
Loaded symbols for /opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so
Reading symbols from /usr/lib64/libcrypto.so.0.9.8...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libcrypto.so.0.9.8
Reading symbols from /usr/lib64/libssl.so.0.9.8...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libssl.so.0.9.8
Reading symbols from /lib/libz.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/libz.so.1
Reading symbols from /opt/couchbase/lib/libv8.so...done.
Loaded symbols for /opt/couchbase/lib/libv8.so
Reading symbols from /lib/libcrypt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/libcrypt.so.1
Reading symbols from /usr/lib/libstdc++.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/libstdc++.so.6
Reading symbols from /lib/libgcc_s.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/libgcc_s.so.1
Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so...done.
Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so
Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so...done.
Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so
Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so...done.
Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so
Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-1b21587-git/priv/lib/couch_icu_driver.so...done.
Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-1b21587-git/priv/lib/couch_icu_driver.so
Reading symbols from /opt/couchbase/lib/libicuuc.so.44...done.
Loaded symbols for /opt/couchbase/lib/libicuuc.so.44
Reading symbols from /opt/couchbase/lib/libicudata.so.44...(no debugging symbols found)...done.
Loaded symbols for /opt/couchbase/lib/libicudata.so.44
Reading symbols from /opt/couchbase/lib/libicui18n.so.44...done.
Loaded symbols for /opt/couchbase/lib/libicui18n.so.44
Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-1b21587-git/priv/lib/couch_ejson_compare.so...done.
Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-1b21587-git/priv/lib/couch_ejson_compare.so
Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/couch_view_parser-1.0/priv/couch_view_parser_nif.so...done.
Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/couch_view_parser-1.0/priv/couch_view_parser_nif.so
Core was generated by `/opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp -A 16 -sbt u -P 327680 -K tru'.
Program terminated with signal 11, Segmentation fault.
#0 right_rotate (root=0x247aa50, blk=0x2495e98) at beam/erl_bestfit_alloc.c:252
252 beam/erl_bestfit_alloc.c: No such file or directory.
in beam/erl_bestfit_alloc.c
(gdb) t a a bt

Thread 28 (Thread 20292):
#0 0x00007f6b4c94aa2d in pthread_mutex_unlock () from /lib/libpthread.so.0
#1 0x000000000043d587 in ethr_mutex_unlock (type=<value optimized out>, unused=<value optimized out>, p=<value optimized out>) at ../include/internal/ethr_mutex.h:597
#2 erts_mtx_unlock (type=<value optimized out>, unused=<value optimized out>, p=<value optimized out>) at beam/erl_threads.h:639
#3 erts_alcu_free_thr_pref (type=<value optimized out>, unused=<value optimized out>, p=<value optimized out>) at beam/erl_alloc_util.c:2950
#4 0x000000000048edf2 in erts_free (offheap=<value optimized out>) at beam/erl_alloc.h:226
#5 erts_bin_free (offheap=<value optimized out>) at beam/erl_binary.h:307
#6 erts_cleanup_offheap (offheap=<value optimized out>) at beam/erl_message.c:174
#7 0x000000000049e991 in delete_process (p=0x7f6b4097b898, pix_lock=<value optimized out>) at beam/erl_process.c:6621
#8 continue_exit_process (p=0x7f6b4097b898, pix_lock=<value optimized out>) at beam/erl_process.c:7663
#9 0x00000000005223c4 in terminate_proc (c_p=0x7f6b4097b898, pc=0x0, reg=0x2432188, bf=0x7f6b46c8d818) at beam/beam_emu.c:5633
#10 handle_error (c_p=0x7f6b4097b898, pc=0x0, reg=0x2432188, bf=0x7f6b46c8d818) at beam/beam_emu.c:5508
#11 0x0000000000525521 in process_main () at beam/beam_emu.c:3273
#12 0x00000000004932a6 in sched_thread_func (vesdp=<value optimized out>) at beam/erl_process.c:3782
#13 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#14 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#15 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#16 0x0000000000000000 in ?? ()

Thread 27 (Thread 20287):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae20704, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 26 (Thread 20285):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae20614, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 25 (Thread 20280):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae203bc, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 24 (Thread 20281):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
--Type <return> to continue, or q <return> to quit--
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae20434, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 23 (Thread 20278):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae202cc, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 22 (Thread 20289):
#0 0x00007f6b4c94f48d in waitpid () from /lib/libpthread.so.0
#1 0x000000000053dd9f in child_waiter (unused=<value optimized out>) at sys/unix/sys.c:2821
#2 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#3 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#4 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 21 (Thread 20276):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae201dc, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 20 (Thread 20275):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae20164, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 19 (Thread 20302):
#0 0x00007f6b4c94f11d in nanosleep () from /lib/libpthread.so.0
#1 0x00007f6b43c026da in terminatorLoop (args=<value optimized out>) at mapreduce_nif.cc:450
#2 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#3 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
--Type <return> to continue, or q <return> to quit--
#4 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 18 (Thread 20286):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae2068c, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 17 (Thread 20282):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae204ac, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 16 (Thread 20274):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae200ec, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 15 (Thread 20293):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x000000000058a983 in wait__ (e=0x2415d20) at pthread/ethr_event.c:185
#2 ethr_event_wait (e=0x2415d20) at pthread/ethr_event.c:218
#3 0x000000000049e11d in erts_tse_wait (fcalls=<value optimized out>, esdp=0x7f6b4b95d200, rq=0x7f6b4b9329c0) at beam/erl_threads.h:1809
#4 scheduler_wait (fcalls=<value optimized out>, esdp=0x7f6b4b95d200, rq=0x7f6b4b9329c0) at beam/erl_process.c:1076
#5 0x00000000004a0156 in schedule (p=<value optimized out>, calls=<value optimized out>) at beam/erl_process.c:5421
#6 0x0000000000522e9f in process_main () at beam/beam_emu.c:1225
#7 0x00000000004932a6 in sched_thread_func (vesdp=<value optimized out>) at beam/erl_process.c:3782
#8 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#9 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#10 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#11 0x0000000000000000 in ?? ()

Thread 14 (Thread 20290):
#0 0x000000000043c225 in mbc_free (allctr=0x2430840, p=0x7f6b2a2ccf30) at beam/erl_alloc_util.c:881
#1 0x000000000043d57f in do_erts_alcu_free (type=<value optimized out>, unused=<value optimized out>, p=<value optimized out>) at beam/erl_alloc_util.c:2898
#2 erts_alcu_free_thr_pref (type=<value optimized out>, unused=<value optimized out>, p=<value optimized out>) at beam/erl_alloc_util.c:2949
#3 0x000000000048edd5 in erts_free (offheap=<value optimized out>) at beam/erl_alloc.h:226
--Type <return> to continue, or q <return> to quit--
#4 erts_bin_free (offheap=<value optimized out>) at beam/erl_binary.h:305
#5 erts_cleanup_offheap (offheap=<value optimized out>) at beam/erl_message.c:174
#6 0x000000000049e991 in delete_process (p=0x7f6b409a3ee8, pix_lock=<value optimized out>) at beam/erl_process.c:6621
#7 continue_exit_process (p=0x7f6b409a3ee8, pix_lock=<value optimized out>) at beam/erl_process.c:7663
#8 0x00000000005223c4 in terminate_proc (c_p=0x7f6b409a3ee8, pc=0x0, reg=0x190, bf=0x7f6b3ecea3c0) at beam/beam_emu.c:5633
#9 handle_error (c_p=0x7f6b409a3ee8, pc=0x0, reg=0x190, bf=0x7f6b3ecea3c0) at beam/beam_emu.c:5508
#10 0x0000000000525521 in process_main () at beam/beam_emu.c:3273
#11 0x00000000004932a6 in sched_thread_func (vesdp=<value optimized out>) at beam/erl_process.c:3782
#12 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#13 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#14 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#15 0x0000000000000000 in ?? ()

Thread 13 (Thread 20268):
#0 0x00007f6b4c94e93d in read () from /lib/libpthread.so.0
#1 0x000000000053cd60 in read (unused=<value optimized out>) at /usr/include/bits/unistd.h:45
#2 signal_dispatcher_thread_func (unused=<value optimized out>) at sys/unix/sys.c:2906
#3 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#4 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#5 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 12 (Thread 20284):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae2059c, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 11 (Thread 20288):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae2077c, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 10 (Thread 20283):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae20524, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

--Type <return> to continue, or q <return> to quit--
Thread 9 (Thread 20269):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x86be54, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x000000000046fb9f in erts_cnd_wait (unused=<value optimized out>) at beam/erl_threads.h:697
#3 erts_smp_cnd_wait (unused=<value optimized out>) at beam/erl_smp.h:472
#4 emergency_watchdog (unused=<value optimized out>) at beam/utils.c:3945
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 8 (Thread 20279):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae20344, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 7 (Thread 20277):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae20254, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 6 (Thread 20262):
#0 0x00007f6b4c493ff3 in select () from /lib/libc.so.6
#1 0x000000000053c780 in erts_sys_main_thread () at sys/unix/sys.c:3049
#2 0x0000000000449c11 in erl_start (argc=54, argv=<value optimized out>) at beam/erl_init.c:1441
#3 0x000000000042b289 in main (argc=0, argv=0x0) at sys/unix/erl_main.c:29

Thread 5 (Thread 20273):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x7f6b4ae20074, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x00000000004ed72b in erts_cnd_wait (arg=<value optimized out>) at beam/erl_threads.h:697
#3 async_get (arg=<value optimized out>) at beam/erl_async.c:207
#4 async_main (arg=<value optimized out>) at beam/erl_async.c:274
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 4 (Thread 25690):
#0 0x00007f6b4c45f39d in nanosleep () from /lib/libc.so.6
#1 0x00007f6b4c494844 in usleep () from /lib/libc.so.6
#2 0x00007f6b46460bc3 in v8::internal::SignalSender::Sleep (this=0x7f6b2c03ef60) at src/platform-linux.cc:1184
#3 v8::internal::SignalSender::Run (this=0x7f6b2c03ef60) at src/platform-linux.cc:1150
--Type <return> to continue, or q <return> to quit--
#4 0x00007f6b4645fe40 in ThreadEntry (arg=0x7f6b2c03ef60) at src/platform-linux.cc:745
#5 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#6 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 3 (Thread 20271):
#0 0x00007f6b4c94b85c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00000000005882a9 in ethr_cond_wait (cnd=0x826204, mtx=0x80) at common/ethr_mutex.c:1350
#2 0x0000000000468399 in erts_cnd_wait (unused=<value optimized out>) at beam/erl_threads.h:697
#3 erts_smp_cnd_wait (unused=<value optimized out>) at beam/erl_smp.h:472
#4 sys_msg_dispatcher_func (unused=<value optimized out>) at beam/erl_trace.c:3238
#5 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#6 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#7 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 2 (Thread 25691):
#0 0x00007f6b4c45f39d in nanosleep () from /lib/libc.so.6
#1 0x00007f6b4c494844 in usleep () from /lib/libc.so.6

#2 0x00007f6b464e3658 in v8::internal::ContextSwitcher::Run (this=0x7f6b2c03b050) at src/v8threads.cc:457
#3 0x00007f6b4645fe40 in ThreadEntry (arg=0x7f6b2c03b050) at src/platform-linux.cc:745
#4 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#5 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 1 (Thread 20291):
#0 right_rotate (root=0x247aa50, blk=0x2495e98) at beam/erl_bestfit_alloc.c:252
#1 tree_insert_fixup (root=0x247aa50, blk=0x2495e98) at beam/erl_bestfit_alloc.c:350
#2 0x000000000044469a in bf_link_free_block (allctr=<value optimized out>, block=0x2495e98, flags=<value optimized out>) at beam/erl_bestfit_alloc.c:764
#3 0x000000000043d57f in do_erts_alcu_free (type=<value optimized out>, unused=<value optimized out>, p=<value optimized out>) at beam/erl_alloc_util.c:2898
#4 erts_alcu_free_thr_pref (type=<value optimized out>, unused=<value optimized out>, p=<value optimized out>) at beam/erl_alloc_util.c:2949
#5 0x0000000000541c20 in destroy (s=0x2495d18) at drivers/common/gzio.c:440
#6 0x0000000000553285 in file_stop (e=<value optimized out>) at drivers/common/efile_drv.c:698
#7 0x000000000048a6a3 in terminate_port (prt=0x7f6b4ab717f8) at beam/io.c:1849
#8 0x000000000048acb8 in erts_do_exit_port (p=0x7f6b4ab717f8, from=<value optimized out>, reason=140098744302144) at beam/io.c:2059
#9 0x000000000049d045 in doit_exit_link (lnk=0x7f6b3f23c810, vpcontext=<value optimized out>) at beam/erl_process.c:7256
#10 0x00000000004ff567 in erts_sweep_links (root=<value optimized out>, doit=0x49cd90 <doit_exit_link>, context=0x7f6b49cf7bd0) at beam/erl_monitors.c:859
#11 0x000000000049e915 in continue_exit_process (p=0x7f6b4098b3d0, pix_lock=<value optimized out>) at beam/erl_process.c:7646
#12 0x00000000005223c4 in terminate_proc (c_p=0x7f6b4098b3d0, pc=0x0, reg=0x2494b68, bf=0x4000) at beam/beam_emu.c:5633
#13 handle_error (c_p=0x7f6b4098b3d0, pc=0x0, reg=0x2494b68, bf=0x4000) at beam/beam_emu.c:5508
#14 0x0000000000525521 in process_main () at beam/beam_emu.c:3273
#15 0x00000000004932a6 in sched_thread_func (vesdp=<value optimized out>) at beam/erl_process.c:3782
#16 0x000000000058a289 in thr_wrapper (vtwd=<value optimized out>) at pthread/ethread.c:106
#17 0x00007f6b4c9469ca in start_thread () from /lib/libpthread.so.0
#18 0x00007f6b4c49b70d in clone () from /lib/libc.so.6
#19 0x0000000000000000 in ?? ()