Cluster run crashes immediatley

I've run this on both my mac and on centos and seen this issue. Rolling back commits fixes this issue.

[jenkins@cen-1722 ns_server]$ make dataclean
rm -rf ./tmp data coverage couch logs tmp
[jenkins@cen-1722 ns_server]$ ./cluster_run
[ns_server:info,2013-04-08T10:20:11.848,babysitter_of_n_0@127.0.0.1:<0.55.0>:ns_babysitter:init_logging:141]Brought up babysitter logging
[ns_server:info,2013-04-08T10:20:11.855,babysitter_of_n_0@127.0.0.1:<0.55.0>:ns_babysitter:log_pending:81]Static config terms:
[

{error_logger_mf_dir,"@DATA_PREFIX@/var/lib/couchbase/logs"}

,

{error_logger_mf_maxbytes,10485760}

,

{error_logger_mf_maxfiles,20}

,

{path_config_bindir,"@BIN_PREFIX@/bin"}

,

{path_config_etcdir,"@BIN_PREFIX@/etc/couchbase"}

,

{path_config_libdir,"@BIN_PREFIX@/lib"}

,

{path_config_datadir,"@DATA_PREFIX@/var/lib/couchbase"}

,

{path_config_tmpdir,"@DATA_PREFIX@/var/lib/couchbase/tmp"}

,

{loglevel_default,debug}

,

{loglevel_couchdb,info}

,

{loglevel_ns_server,debug}

,

{loglevel_error_logger,debug}

,

{loglevel_user,debug}

,

{loglevel_menelaus,debug}

,

{loglevel_ns_doctor,debug}

,

{loglevel_stats,debug}

,

{loglevel_rebalance,debug}

,

{loglevel_cluster,debug}

,

{loglevel_views,debug}

,

{loglevel_mapreduce_errors,debug}

,

{loglevel_xdcr,debug}

]
[ns_server:warn,2013-04-08T10:20:11.855,babysitter_of_n_0@127.0.0.1:<0.55.0>:ns_babysitter:log_pending:81]not overriding parameter error_logger_mf_dir, which is given from command line
[ns_server:warn,2013-04-08T10:20:11.856,babysitter_of_n_0@127.0.0.1:<0.55.0>:ns_babysitter:log_pending:81]not overriding parameter error_logger_mf_maxbytes, which is given from command line
[ns_server:warn,2013-04-08T10:20:11.856,babysitter_of_n_0@127.0.0.1:<0.55.0>:ns_babysitter:log_pending:81]not overriding parameter error_logger_mf_maxfiles, which is given from command line
[ns_server:warn,2013-04-08T10:20:11.856,babysitter_of_n_0@127.0.0.1:<0.55.0>:ns_babysitter:log_pending:81]not overriding parameter path_config_bindir, which is given from command line
[ns_server:warn,2013-04-08T10:20:11.856,babysitter_of_n_0@127.0.0.1:<0.55.0>:ns_babysitter:log_pending:81]not overriding parameter path_config_etcdir, which is given from command line
[ns_server:warn,2013-04-08T10:20:11.856,babysitter_of_n_0@127.0.0.1:<0.55.0>:ns_babysitter:log_pending:81]not overriding parameter path_config_libdir, which is given from command line
[ns_server:warn,2013-04-08T10:20:11.857,babysitter_of_n_0@127.0.0.1:<0.55.0>:ns_babysitter:log_pending:81]not overriding parameter path_config_datadir, which is given from command line
[ns_server:warn,2013-04-08T10:20:11.857,babysitter_of_n_0@127.0.0.1:<0.55.0>:ns_babysitter:log_pending:81]not overriding parameter path_config_tmpdir, which is given from command line
[ns_server:warn,2013-04-08T10:20:11.859,babysitter_of_n_0@127.0.0.1:<0.55.0>:ns_babysitter:start:44]Could not lock myself into a memory:

{error,enomem}

. Ignoring.
[ns_server:info,2013-04-08T10:20:11.862,babysitter_of_n_0@127.0.0.1:<0.55.0>:ns_babysitter:start:65]babysitter cookie: tadfclmbivtghwyt

[ns_server:info,2013-04-08T10:20:11.872,babysitter_of_n_0@127.0.0.1:ns_crash_log<0.75.0>:ns_crash_log:init:43]crash_log.bin path: data/n_0/logs/crash_log.bin
[ns_server:warn,2013-04-08T10:20:11.873,babysitter_of_n_0@127.0.0.1:ns_crash_log<0.75.0>:ns_crash_log:read_crash_log:135]Couldn't load crash_log from data/n_0/logs/crash_log.bin (perhaps it's first startup):

{error, enoent}

[error_logger:info,2013-04-08T10:20:11.873,babysitter_of_n_0@127.0.0.1:error_logger<0.6.0>:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor:

{local,ns_babysitter_sup}
started: [{pid,<0.75.0>},
{name,ns_crash_log},
{mfargs,{ns_crash_log,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]

[ns_server:debug,2013-04-08T10:20:11.875,babysitter_of_n_0@127.0.0.1:<0.76.0>:supervisor_cushion:init:43]starting ns_port_server with delay of 5000
[error_logger:error,2013-04-08T10:20:11.888,babysitter_of_n_0@127.0.0.1:error_logger<0.6.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_port_server:init/1
pid: <0.77.0>
registered_name: []
exception exit: {badarg,
[{erlang,open_port,
[{spawn_executable, "/usr/local/lib/erlang/bin/erl"},
[ns_server_no_stderr_to_stdout,exit_status,
use_stdio,stream,eof,
{args, ["+A","16","-smp","enable","+sbt","u","+P", "327680","+K","true","-setcookie", "nocookie","-kernel", "inet_dist_listen_min","21100", "inet_dist_listen_max","21299", "error_logger","false","-sasl", "sasl_error_logger","false","-nouser", "-ns_server","babysitter_cookie", "'tadfclmbivtghwyt'","-run", "child_erlang","child_start", "ns_bootstrap","--","-pa", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/snappy-1.0.4/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/mapreduce-1.0/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/etap/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/ejson-0.1.0/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/erlang-oauth/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch_view_parser-1.0/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/mochiweb-1.4.1/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch_set_view-1.0.0/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch_index_merger-1.0.0/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/lhttpc-1.3/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch-1.2.0a-94d2593-git/ebin", "./deps/ale/ebin","./deps/mlockall/ebin", "./deps/ns_babysitter/ebin", "./deps/gen_smtp/ebin", "./deps/erlwsh/ebin","./ebin", "/home/jenkins/couchbase/install/lib/couchdb/plugins/geocouch/ebin", "-kernel","inet_dist_listen_min","21100", "inet_dist_listen_max","21199", "error_logger","false","-sasl", "sasl_error_logger","false","-couch_ini", "/home/jenkins/couchbase/install/etc/couchdb/default.ini", "/home/jenkins/couchbase/install/etc/couchdb/default.d/capi.ini", "/home/jenkins/couchbase/install/etc/couchdb/default.d/geocouch.ini", "couch/n_0_conf.ini","-ns_server", "config_path","\"etc/static_config.in\"", "-ns_server","enable_mlockall","true", "error_logger_mf_dir","\"logs/n_0\"", "error_logger_mf_maxbytes","10485760", "error_logger_mf_maxfiles","10", "dont_suppress_stderr_logger","true", "path_config_etcdir","\"priv\"", "path_config_bindir", "\"/home/jenkins/couchbase/install/bin\"", "path_config_libdir", "\"/home/jenkins/couchbase/install/lib\"", "path_config_datadir","\"data/n_0\"", "path_config_tmpdir","\"tmp/\"", "rest_port","9000","mccouch_port","13000", "memcached_port","12000","moxi_port", "12001","memcached_dedicated_port", "11999","short_name","\"n_0\"", "rename_ip","\"10.5.2.33\"","-ns_server", "loglevel_stderr","debug"]},
exit_status,
{line,8192},
stderr_to_stdout]],
[]},
{ns_port_server,open_port,1,
[{file,"src/ns_port_server.erl"},{line,189}]},
{ns_port_server,init,1,
[{file,"src/ns_port_server.erl"},{line,75}]},
{gen_server,init_it,6,
[{file,"gen_server.erl"},{line,297}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,227}]}]}
in function gen_server:init_it/6 (gen_server.erl, line 321)
ancestors: [<0.76.0>,ns_babysitter_sup,<0.55.0>]
messages: []
links: [<0.76.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 6765
stack_size: 24
reductions: 132
neighbours:

[error_logger:error,2013-04-08T10:20:11.901,babysitter_of_n_0@127.0.0.1:error_logger<0.6.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: supervisor_cushion:init/1
pid: <0.76.0>
registered_name: []
exception exit: {undef,
[{'ale_logger-user',xwarn,
[supervisor_cushion,die_slowly,81,
{supervisor_cushion,1},
"Service ~p exited on node ~p in ~.2fs~n",
[ns_server,
{'_susp',#Ref<0.0.0.201>, #Fun<supervisor_cushion.0.118798826>},
1.0e-6]],
[]},
{supervisor_cushion,die_slowly,2,
[{file,"src/supervisor_cushion.erl"},{line,81}]},
{supervisor_cushion,init,1,
[{file,"src/supervisor_cushion.erl"},{line,49}]},
{gen_server,init_it,6,
[{file,"gen_server.erl"},{line,297}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,227}]}]}
in function gen_server:init_it/6 (gen_server.erl, line 321)
ancestors: [ns_babysitter_sup,<0.55.0>]
messages: [{'EXIT',<0.77.0>,
{badarg,
[{erlang,open_port,
[{spawn_executable, "/usr/local/lib/erlang/bin/erl"},
[ns_server_no_stderr_to_stdout,exit_status,
use_stdio,stream,eof,
{args, ["+A","16","-smp","enable","+sbt","u", "+P","327680","+K","true","-setcookie", "nocookie","-kernel", "inet_dist_listen_min","21100", "inet_dist_listen_max","21299", "error_logger","false","-sasl", "sasl_error_logger","false","-nouser", "-ns_server","babysitter_cookie", "'tadfclmbivtghwyt'","-run", "child_erlang","child_start", "ns_bootstrap","--","-pa", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/snappy-1.0.4/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/mapreduce-1.0/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/etap/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/ejson-0.1.0/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/erlang-oauth/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch_view_parser-1.0/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/mochiweb-1.4.1/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch_set_view-1.0.0/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch_index_merger-1.0.0/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/lhttpc-1.3/ebin", "/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch-1.2.0a-94d2593-git/ebin", "./deps/ale/ebin", "./deps/mlockall/ebin", "./deps/ns_babysitter/ebin", "./deps/gen_smtp/ebin", "./deps/erlwsh/ebin","./ebin", "/home/jenkins/couchbase/install/lib/couchdb/plugins/geocouch/ebin", "-kernel","inet_dist_listen_min", "21100","inet_dist_listen_max","21199", "error_logger","false","-sasl", "sasl_error_logger","false", "-couch_ini", "/home/jenkins/couchbase/install/etc/couchdb/default.ini", "/home/jenkins/couchbase/install/etc/couchdb/default.d/capi.ini", "/home/jenkins/couchbase/install/etc/couchdb/default.d/geocouch.ini", "couch/n_0_conf.ini","-ns_server", "config_path", "\"etc/static_config.in\"","-ns_server", "enable_mlockall","true", "error_logger_mf_dir","\"logs/n_0\"", "error_logger_mf_maxbytes","10485760", "error_logger_mf_maxfiles","10", "dont_suppress_stderr_logger","true", "path_config_etcdir","\"priv\"", "path_config_bindir", "\"/home/jenkins/couchbase/install/bin\"", "path_config_libdir", "\"/home/jenkins/couchbase/install/lib\"", "path_config_datadir","\"data/n_0\"", "path_config_tmpdir","\"tmp/\"", "rest_port","9000","mccouch_port", "13000","memcached_port","12000", "moxi_port","12001", "memcached_dedicated_port","11999", "short_name","\"n_0\"","rename_ip", "\"10.5.2.33\"","-ns_server", "loglevel_stderr","debug"]},
exit_status,
{line,8192},
stderr_to_stdout]],
[]},
{ns_port_server,open_port,1,
[{file,"src/ns_port_server.erl"},{line,189}]},
{ns_port_server,init,1,
[{file,"src/ns_port_server.erl"},{line,75}]},
{gen_server,init_it,6,
[{file,"gen_server.erl"},{line,297}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,227}]}]}}]
links: [<0.74.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 17711
stack_size: 24
reductions: 1699
neighbours:

[error_logger:error,2013-04-08T10:20:11.906,babysitter_of_n_0@127.0.0.1:error_logger<0.6.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_babysitter_sup}

Context: start_error
Reason: {undef,
[{'ale_logger-user',xwarn,
[supervisor_cushion,die_slowly,81,

{supervisor_cushion,1}

,
"Service ~p exited on node ~p in ~.2fs~n",
[ns_server,

{'_susp',#Ref<0.0.0.201>, #Fun<supervisor_cushion.0.118798826>}

,
1.0e-6]],
[]},
{supervisor_cushion,die_slowly,2,
[

{file,"src/supervisor_cushion.erl"},{line,81}]},
{supervisor_cushion,init,1,
[{file,"src/supervisor_cushion.erl"}

,

{line,49}

]},
{gen_server,init_it,6,
[

{file,"gen_server.erl"}

,

{line,297}

]},
{proc_lib,init_p_do_apply,3,
[

{file,"proc_lib.erl"}

,

{line,227}

]}]}
Offender: [

{pid,undefined}

,
{name,
{ns_server,"/usr/local/lib/erlang/bin/erl",
["+A","16","-smp","enable","+sbt","u","+P","327680",
"+K","true","-setcookie","nocookie","-kernel",
"inet_dist_listen_min","21100",
"inet_dist_listen_max","21299","error_logger",
"false","-sasl","sasl_error_logger","false",
"-nouser","-ns_server","babysitter_cookie",
"'tadfclmbivtghwyt'","-run","child_erlang",
"child_start","ns_bootstrap","--","-pa",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/snappy-1.0.4/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/mapreduce-1.0/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/etap/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/ejson-0.1.0/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/erlang-oauth/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch_view_parser-1.0/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/mochiweb-1.4.1/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch_set_view-1.0.0/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch_index_merger-1.0.0/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/lhttpc-1.3/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch-1.2.0a-94d2593-git/ebin",
"./deps/ale/ebin","./deps/mlockall/ebin",
"./deps/ns_babysitter/ebin","./deps/gen_smtp/ebin",
"./deps/erlwsh/ebin","./ebin",
"/home/jenkins/couchbase/install/lib/couchdb/plugins/geocouch/ebin",
"-kernel","inet_dist_listen_min","21100",
"inet_dist_listen_max","21199","error_logger",
"false","-sasl","sasl_error_logger","false",
"-couch_ini",
"/home/jenkins/couchbase/install/etc/couchdb/default.ini",
"/home/jenkins/couchbase/install/etc/couchdb/default.d/capi.ini",
"/home/jenkins/couchbase/install/etc/couchdb/default.d/geocouch.ini",
"couch/n_0_conf.ini","-ns_server","config_path",
"\"etc/static_config.in\"","-ns_server",
"enable_mlockall","true","error_logger_mf_dir",
"\"logs/n_0\"","error_logger_mf_maxbytes",
"10485760","error_logger_mf_maxfiles","10",
"dont_suppress_stderr_logger","true",
"path_config_etcdir","\"priv\"",
"path_config_bindir",
"\"/home/jenkins/couchbase/install/bin\"",
"path_config_libdir",
"\"/home/jenkins/couchbase/install/lib\"",
"path_config_datadir","\"data/n_0\"",
"path_config_tmpdir","\"tmp/\"","rest_port","9000",
"mccouch_port","13000","memcached_port","12000",
"moxi_port","12001","memcached_dedicated_port",
"11999","short_name","\"n_0\"","rename_ip",
"\"10.5.2.33\"","-ns_server","loglevel_stderr",
"debug"],
[ns_server_no_stderr_to_stdout,port_server_send_eol,
exit_status,use_stdio,stream,eof]}},
{mfargs,
{supervisor_cushion,start_link,
[ns_server,5000,1000000,ns_port_server,start_link,
[ns_server,"/usr/local/lib/erlang/bin/erl",
["+A","16","-smp","enable","+sbt","u","+P",
"327680","+K","true","-setcookie","nocookie",
"-kernel","inet_dist_listen_min","21100",
"inet_dist_listen_max","21299","error_logger",
"false","-sasl","sasl_error_logger","false",
"-nouser","-ns_server","babysitter_cookie",
"'tadfclmbivtghwyt'","-run","child_erlang",
"child_start","ns_bootstrap","--","-pa",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/snappy-1.0.4/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/mapreduce-1.0/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/etap/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/ejson-0.1.0/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/erlang-oauth/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch_view_parser-1.0/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/mochiweb-1.4.1/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch_set_view-1.0.0/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch_index_merger-1.0.0/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/lhttpc-1.3/ebin",
"/home/jenkins/couchbase/install/lib/couchdb/erlang/lib/couch-1.2.0a-94d2593-git/ebin",
"./deps/ale/ebin","./deps/mlockall/ebin",
"./deps/ns_babysitter/ebin",
"./deps/gen_smtp/ebin","./deps/erlwsh/ebin",
"./ebin",
"/home/jenkins/couchbase/install/lib/couchdb/plugins/geocouch/ebin",
"-kernel","inet_dist_listen_min","21100",
"inet_dist_listen_max","21199","error_logger",
"false","-sasl","sasl_error_logger","false",
"-couch_ini",
"/home/jenkins/couchbase/install/etc/couchdb/default.ini",
"/home/jenkins/couchbase/install/etc/couchdb/default.d/capi.ini",
"/home/jenkins/couchbase/install/etc/couchdb/default.d/geocouch.ini",
"couch/n_0_conf.ini","-ns_server","config_path",
"\"etc/static_config.in\"","-ns_server",
"enable_mlockall","true","error_logger_mf_dir",
"\"logs/n_0\"","error_logger_mf_maxbytes",
"10485760","error_logger_mf_maxfiles","10",
"dont_suppress_stderr_logger","true",
"path_config_etcdir","\"priv\"",
"path_config_bindir",
"\"/home/jenkins/couchbase/install/bin\"",
"path_config_libdir",
"\"/home/jenkins/couchbase/install/lib\"",
"path_config_datadir","\"data/n_0\"",
"path_config_tmpdir","\"tmp/\"","rest_port",
"9000","mccouch_port","13000","memcached_port",
"12000","moxi_port","12001",
"memcached_dedicated_port","11999","short_name",
"\"n_0\"","rename_ip","\"10.5.2.33\"",
"-ns_server","loglevel_stderr","debug"],
[ns_server_no_stderr_to_stdout,
port_server_send_eol,exit_status,use_stdio,
stream,eof]]]}},

{restart_type,permanent}

,

{shutdown,86400000}

,

{child_type,worker}

]

[error_logger:info,2013-04-08T10:20:11.914,babysitter_of_n_0@127.0.0.1:error_logger<0.6.0>:ale_error_logger_handler:log_report:72]
=========================INFO REPORT=========================
application: ns_babysitter
exited: {shutdown,{ns_babysitter,start,[normal,[]]}}
type: temporary

Crap error:{badmatch,{error,{shutdown,{ns_babysitter,start,[normal,[]]}}}}
[{child_erlang,do_child_start,1,[

{file,"src/child_erlang.erl"},{line,54}]},
{child_erlang,child_start,1,[{file,"src/child_erlang.erl"}

,

{line,32}]},
{init,start_it,1,[]},
{init,start_em,1,[]}]
[ns_server:debug,2013-04-08T10:20:12.399,babysitter_of_n_0@127.0.0.1:<0.2.0>:child_erlang:child_start:35]Crap to start: error:{badmatch,
{error,
{shutdown,{ns_babysitter,start,[normal,[]]}}}}
[{child_erlang,do_child_start,1,[{file,"src/child_erlang.erl"},{line,54}]},
{child_erlang,child_start,1,[{file,"src/child_erlang.erl"},{line,32}

]},

{init,start_it,1,[]}

,

{init,start_em,1,[]}

]

('Closing %d\n', 4)