==============================================================================
couchbase logs (error)
cbbrowse_logs error
==============================================================================
[ns_server:error,2012-11-06T18:06:03.635,ns_1@127.0.0.1:ns_heart<0.374.0>:ns_heart:grab_samples_loading_tasks:319]Failed to grab samples loader tasks: {exit,
                                      {noproc,
                                       {gen_server,call,
                                        [samples_loader_tasks,get_tasks,
                                         2000]}},
                                      [{gen_server,call,3},
                                       {ns_heart,grab_samples_loading_tasks,0},
                                       {ns_heart,current_status,0},
                                       {ns_heart,handle_info,2},
                                       {gen_server,handle_msg,5},
                                       {proc_lib,init_p_do_apply,3}]}
[ns_server:error,2012-11-06T18:06:03.642,ns_1@127.0.0.1:ns_heart<0.374.0>:ns_heart:grab_samples_loading_tasks:319]Failed to grab samples loader tasks: {exit,
                                      {noproc,
                                       {gen_server,call,
                                        [samples_loader_tasks,get_tasks,
                                         2000]}},
                                      [{gen_server,call,3},
                                       {ns_heart,grab_samples_loading_tasks,0},
                                       {ns_heart,current_status,0},
                                       {ns_heart,handle_call,3},
                                       {gen_server,handle_msg,5},
                                       {proc_lib,init_p_do_apply,3}]}
[error_logger:error,2012-11-06T18:06:27.072,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
     Supervisor: {local,ns_bucket_sup}
     Context:    shutdown_error
     Reason:     normal
     Offender:   [{pid,<0.453.0>},
                  {name,buckets_observing_subscription},
                  {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}},
                  {restart_type,permanent},
                  {shutdown,1000},
                  {child_type,worker}]


[error_logger:error,2012-11-06T18:06:27.960,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
     Supervisor: {local,ns_server_cluster_sup}
     Context:    shutdown_error
     Reason:     killed
     Offender:   [{pid,<0.345.0>},
                  {name,vbucket_filter_changes_registry},
                  {mfargs,
                      {ns_process_registry,start_link,
                          [vbucket_filter_changes_registry]}},
                  {restart_type,permanent},
                  {shutdown,100},
                  {child_type,worker}]


[error_logger:error,2012-11-06T18:06:32.960,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
     Supervisor: {local,mb_mnesia_sup}
     Context:    shutdown_error
     Reason:     killed
     Offender:   [{pid,<0.283.0>},
                  {name,mb_mnesia},
                  {mfargs,{mb_mnesia,start_link,[]}},
                  {restart_type,permanent},
                  {shutdown,5000},
                  {child_type,worker}]


[error_logger:error,2012-11-06T18:06:33.138,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Generic server <0.232.0> terminating 
** Last message in was {'EXIT',<0.218.0>,killed}
** When Server state == {db,<0.232.0>,<0.233.0>,nil,<<"1352225162344489">>,
                            <0.228.0>,<0.234.0>,
                            {db_header,11,1,
                                <<0,0,0,0,13,103,0,0,0,0,0,51,0,0,0,0,1,0,0,0,
                                  0,0,0,0,0,0,13,69>>,
                                <<0,0,0,0,13,154,0,0,0,0,0,49,0,0,0,0,1>>,
                                nil,0,nil,nil},
                            1,
                            {btree,<0.228.0>,
                                {3431,
                                 <<0,0,0,0,1,0,0,0,0,0,0,0,0,0,13,69>>,
                                 51},
                                #Fun<couch_db_updater.7.89001503>,
                                #Fun<couch_db_updater.8.75953275>,
                                #Fun<couch_btree.5.72034400>,
                                #Fun<couch_db_updater.9.14108461>,1279,2558,
                                true},
                            {btree,<0.228.0>,
                                {3482,<<0,0,0,0,1>>,49},
                                #Fun<couch_db_updater.10.50603258>,
                                #Fun<couch_db_updater.11.85949495>,
                                #Fun<couch_db_updater.6.41937156>,
                                #Fun<couch_db_updater.12.107260449>,1279,2558,
                                true},
                            {btree,<0.228.0>,nil,
                                #Fun<couch_btree.3.59827385>,
                                #Fun<couch_btree.4.7841881>,
                                #Fun<couch_btree.5.72034400>,nil,1279,2558,
                                true},
                            1,<<"_users">>,
                            "/opt/couchbase/var/lib/couchbase/data/_users.couch.1",
                            [],nil,
                            {user_ctx,null,[],undefined},
                            nil,
                            [before_header,after_header,on_file_open],
                            [create,
                             {user_ctx,
                                 {user_ctx,null,[<<"_admin">>],undefined}},
                             sys_db]}
** Reason for termination == 
** killed

[error_logger:error,2012-11-06T18:06:33.143,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
  crasher:
    initial call: couch_db:init/1
    pid: <0.232.0>
    registered_name: []
    exception exit: killed
      in function  gen_server:terminate/6
    ancestors: [couch_server,couch_primary_services,couch_server_sup,
                  cb_couch_sup,ns_server_cluster_sup,<0.58.0>]
    messages: []
    links: []
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 987
    stack_size: 24
    reductions: 284
  neighbours:

[error_logger:error,2012-11-06T18:06:33.144,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Generic server <0.446.0> terminating 
** Last message in was {'EXIT',<0.218.0>,killed}
** When Server state == {db,<0.446.0>,<0.447.0>,nil,<<"1352225163681064">>,
                            <0.442.0>,<0.449.0>,
                            {db_header,11,1,
                                <<0,0,0,0,11,84,0,0,0,0,0,62,0,0,0,0,1,0,0,0,0,
                                  0,0,0,0,0,11,50>>,
                                <<0,0,0,0,11,146,0,0,0,0,0,60,0,0,0,0,1>>,
                                nil,0,nil,nil},
                            1,
                            {btree,<0.442.0>,
                                {2900,
                                 <<0,0,0,0,1,0,0,0,0,0,0,0,0,0,11,50>>,
                                 62},
                                #Fun<couch_db_updater.7.89001503>,
                                #Fun<couch_db_updater.8.75953275>,
                                #Fun<couch_btree.5.72034400>,
                                #Fun<couch_db_updater.9.14108461>,1279,2558,
                                true},
                            {btree,<0.442.0>,
                                {2962,<<0,0,0,0,1>>,60},
                                #Fun<couch_db_updater.10.50603258>,
                                #Fun<couch_db_updater.11.85949495>,
                                #Fun<couch_db_updater.6.41937156>,
                                #Fun<couch_db_updater.12.107260449>,1279,2558,
                                true},
                            {btree,<0.442.0>,nil,
                                #Fun<couch_btree.3.59827385>,
                                #Fun<couch_btree.4.7841881>,
                                #Fun<couch_btree.5.72034400>,nil,1279,2558,
                                true},
                            1,<<"_replicator">>,
                            "/opt/couchbase/var/lib/couchbase/data/_replicator.couch.1",
                            [],nil,
                            {user_ctx,null,[],undefined},
                            nil,
                            [before_header,after_header,on_file_open],
                            [create,sys_db,
                             {user_ctx,
                                 {user_ctx,null,
                                     [<<"_admin">>,<<"_replicator">>],
                                     undefined}}]}
** Reason for termination == 
** killed

[error_logger:error,2012-11-06T18:06:33.144,ns_1@127.0.0.1:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
  crasher:
    initial call: couch_db:init/1
    pid: <0.446.0>
    registered_name: []
    exception exit: killed
      in function  gen_server:terminate/6
    ancestors: [couch_server,couch_primary_services,couch_server_sup,
                  cb_couch_sup,ns_server_cluster_sup,<0.58.0>]
    messages: []
    links: []
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 987
    stack_size: 24
    reductions: 313
  neighbours:

[ns_server:error,2012-11-06T18:06:38.848,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:ns_heart<0.375.0>:ns_heart:grab_samples_loading_tasks:319]Failed to grab samples loader tasks: {exit,
                                      {noproc,
                                       {gen_server,call,
                                        [samples_loader_tasks,get_tasks,
                                         2000]}},
                                      [{gen_server,call,3},
                                       {ns_heart,grab_samples_loading_tasks,0},
                                       {ns_heart,current_status,0},
                                       {ns_heart,handle_info,2},
                                       {gen_server,handle_msg,5},
                                       {proc_lib,init_p_do_apply,3}]}
[ns_server:error,2012-11-06T18:06:38.855,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:ns_heart<0.375.0>:ns_heart:grab_samples_loading_tasks:319]Failed to grab samples loader tasks: {exit,
                                      {noproc,
                                       {gen_server,call,
                                        [samples_loader_tasks,get_tasks,
                                         2000]}},
                                      [{gen_server,call,3},
                                       {ns_heart,grab_samples_loading_tasks,0},
                                       {ns_heart,current_status,0},
                                       {ns_heart,handle_call,3},
                                       {gen_server,handle_msg,5},
                                       {proc_lib,init_p_do_apply,3}]}
[error_logger:error,2012-11-06T18:06:45.856,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
     Supervisor: {local,ns_bucket_sup}
     Context:    shutdown_error
     Reason:     normal
     Offender:   [{pid,<0.454.0>},
                  {name,buckets_observing_subscription},
                  {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}},
                  {restart_type,permanent},
                  {shutdown,1000},
                  {child_type,worker}]


[error_logger:error,2012-11-06T18:06:46.152,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
     Supervisor: {local,ns_server_cluster_sup}
     Context:    shutdown_error
     Reason:     killed
     Offender:   [{pid,<0.346.0>},
                  {name,vbucket_filter_changes_registry},
                  {mfargs,
                      {ns_process_registry,start_link,
                          [vbucket_filter_changes_registry]}},
                  {restart_type,permanent},
                  {shutdown,100},
                  {child_type,worker}]


[error_logger:error,2012-11-06T18:06:46.455,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Generic server <0.447.0> terminating 
** Last message in was {'EXIT',<0.225.0>,killed}
** When Server state == {db,<0.447.0>,<0.448.0>,nil,<<"1352225198888361">>,
                            <0.443.0>,<0.449.0>,
                            {db_header,11,1,
                                <<0,0,0,0,11,84,0,0,0,0,0,62,0,0,0,0,1,0,0,0,0,
                                  0,0,0,0,0,11,50>>,
                                <<0,0,0,0,11,146,0,0,0,0,0,60,0,0,0,0,1>>,
                                nil,0,nil,nil},
                            1,
                            {btree,<0.443.0>,
                                {2900,
                                 <<0,0,0,0,1,0,0,0,0,0,0,0,0,0,11,50>>,
                                 62},
                                #Fun<couch_db_updater.7.89001503>,
                                #Fun<couch_db_updater.8.75953275>,
                                #Fun<couch_btree.5.72034400>,
                                #Fun<couch_db_updater.9.14108461>,1279,2558,
                                true},
                            {btree,<0.443.0>,
                                {2962,<<0,0,0,0,1>>,60},
                                #Fun<couch_db_updater.10.50603258>,
                                #Fun<couch_db_updater.11.85949495>,
                                #Fun<couch_db_updater.6.41937156>,
                                #Fun<couch_db_updater.12.107260449>,1279,2558,
                                true},
                            {btree,<0.443.0>,nil,
                                #Fun<couch_btree.3.59827385>,
                                #Fun<couch_btree.4.7841881>,
                                #Fun<couch_btree.5.72034400>,nil,1279,2558,
                                true},
                            1,<<"_replicator">>,
                            "/opt/couchbase/var/lib/couchbase/data/_replicator.couch.1",
                            [],nil,
                            {user_ctx,null,[],undefined},
                            nil,
                            [before_header,after_header,on_file_open],
                            [create,sys_db,
                             {user_ctx,
                                 {user_ctx,null,
                                     [<<"_admin">>,<<"_replicator">>],
                                     undefined}}]}
** Reason for termination == 
** killed

[error_logger:error,2012-11-06T18:06:46.457,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
  crasher:
    initial call: couch_db:init/1
    pid: <0.447.0>
    registered_name: []
    exception exit: killed
      in function  gen_server:terminate/6
    ancestors: [couch_server,couch_primary_services,couch_server_sup,
                  cb_couch_sup,ns_server_cluster_sup,<0.64.0>]
    messages: []
    links: []
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 987
    stack_size: 24
    reductions: 305
  neighbours:

[error_logger:error,2012-11-06T18:06:46.472,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Generic server <0.239.0> terminating 
** Last message in was {'EXIT',<0.225.0>,killed}
** When Server state == {db,<0.239.0>,<0.240.0>,nil,<<"1352225197571443">>,
                            <0.235.0>,<0.241.0>,
                            {db_header,11,1,
                                <<0,0,0,0,13,103,0,0,0,0,0,51,0,0,0,0,1,0,0,0,
                                  0,0,0,0,0,0,13,69>>,
                                <<0,0,0,0,13,154,0,0,0,0,0,49,0,0,0,0,1>>,
                                nil,0,nil,nil},
                            1,
                            {btree,<0.235.0>,
                                {3431,
                                 <<0,0,0,0,1,0,0,0,0,0,0,0,0,0,13,69>>,
                                 51},
                                #Fun<couch_db_updater.7.89001503>,
                                #Fun<couch_db_updater.8.75953275>,
                                #Fun<couch_btree.5.72034400>,
                                #Fun<couch_db_updater.9.14108461>,1279,2558,
                                true},
                            {btree,<0.235.0>,
                                {3482,<<0,0,0,0,1>>,49},
                                #Fun<couch_db_updater.10.50603258>,
                                #Fun<couch_db_updater.11.85949495>,
                                #Fun<couch_db_updater.6.41937156>,
                                #Fun<couch_db_updater.12.107260449>,1279,2558,
                                true},
                            {btree,<0.235.0>,nil,
                                #Fun<couch_btree.3.59827385>,
                                #Fun<couch_btree.4.7841881>,
                                #Fun<couch_btree.5.72034400>,nil,1279,2558,
                                true},
                            1,<<"_users">>,
                            "/opt/couchbase/var/lib/couchbase/data/_users.couch.1",
                            [],nil,
                            {user_ctx,null,[],undefined},
                            nil,
                            [before_header,after_header,on_file_open],
                            [create,
                             {user_ctx,
                                 {user_ctx,null,[<<"_admin">>],undefined}},
                             sys_db]}
** Reason for termination == 
** killed

[error_logger:error,2012-11-06T18:06:46.475,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
  crasher:
    initial call: couch_db:init/1
    pid: <0.239.0>
    registered_name: []
    exception exit: killed
      in function  gen_server:terminate/6
    ancestors: [couch_server,couch_primary_services,couch_server_sup,
                  cb_couch_sup,ns_server_cluster_sup,<0.64.0>]
    messages: []
    links: []
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 987
    stack_size: 24
    reductions: 284
  neighbours:

[ns_server:error,2012-11-06T18:06:47.663,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:ns_heart<0.731.0>:ns_heart:grab_samples_loading_tasks:319]Failed to grab samples loader tasks: {exit,
                                      {noproc,
                                       {gen_server,call,
                                        [samples_loader_tasks,get_tasks,
                                         2000]}},
                                      [{gen_server,call,3},
                                       {ns_heart,grab_samples_loading_tasks,0},
                                       {ns_heart,current_status,0},
                                       {ns_heart,handle_info,2},
                                       {gen_server,handle_msg,5},
                                       {proc_lib,init_p_do_apply,3}]}
[ns_server:error,2012-11-06T18:06:47.666,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:ns_heart<0.731.0>:ns_heart:grab_samples_loading_tasks:319]Failed to grab samples loader tasks: {exit,
                                      {noproc,
                                       {gen_server,call,
                                        [samples_loader_tasks,get_tasks,
                                         2000]}},
                                      [{gen_server,call,3},
                                       {ns_heart,grab_samples_loading_tasks,0},
                                       {ns_heart,current_status,0},
                                       {ns_heart,handle_call,3},
                                       {gen_server,handle_msg,5},
                                       {proc_lib,init_p_do_apply,3}]}
[error_logger:error,2012-11-06T18:06:52.663,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
     Supervisor: {local,ns_bucket_sup}
     Context:    shutdown_error
     Reason:     normal
     Offender:   [{pid,<0.802.0>},
                  {name,buckets_observing_subscription},
                  {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}},
                  {restart_type,permanent},
                  {shutdown,1000},
                  {child_type,worker}]


[ns_server:error,2012-11-06T18:06:52.663,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:ns_heart<0.731.0>:ns_heart:grab_samples_loading_tasks:319]Failed to grab samples loader tasks: {exit,
                                      {noproc,
                                       {gen_server,call,
                                        [samples_loader_tasks,get_tasks,
                                         2000]}},
                                      [{gen_server,call,3},
                                       {ns_heart,grab_samples_loading_tasks,0},
                                       {ns_heart,current_status,0},
                                       {ns_heart,handle_info,2},
                                       {gen_server,handle_msg,5},
                                       {proc_lib,init_p_do_apply,3}]}
[error_logger:error,2012-11-06T18:06:52.926,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
     Supervisor: {local,ns_server_cluster_sup}
     Context:    shutdown_error
     Reason:     killed
     Offender:   [{pid,<0.704.0>},
                  {name,vbucket_filter_changes_registry},
                  {mfargs,
                      {ns_process_registry,start_link,
                          [vbucket_filter_changes_registry]}},
                  {restart_type,permanent},
                  {shutdown,100},
                  {child_type,worker}]


[error_logger:error,2012-11-06T18:06:57.933,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
     Supervisor: {local,mb_mnesia_sup}
     Context:    shutdown_error
     Reason:     killed
     Offender:   [{pid,<0.608.0>},
                  {name,mb_mnesia},
                  {mfargs,{mb_mnesia,start_link,[]}},
                  {restart_type,permanent},
                  {shutdown,5000},
                  {child_type,worker}]


[error_logger:error,2012-11-06T18:06:58.075,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Generic server <0.796.0> terminating 
** Last message in was {'EXIT',<0.555.0>,killed}
** When Server state == {db,<0.796.0>,<0.797.0>,nil,<<"1352225207678085">>,
                            <0.792.0>,<0.798.0>,
                            {db_header,11,1,
                                <<0,0,0,0,11,84,0,0,0,0,0,62,0,0,0,0,1,0,0,0,0,
                                  0,0,0,0,0,11,50>>,
                                <<0,0,0,0,11,146,0,0,0,0,0,60,0,0,0,0,1>>,
                                nil,0,nil,nil},
                            1,
                            {btree,<0.792.0>,
                                {2900,
                                 <<0,0,0,0,1,0,0,0,0,0,0,0,0,0,11,50>>,
                                 62},
                                #Fun<couch_db_updater.7.89001503>,
                                #Fun<couch_db_updater.8.75953275>,
                                #Fun<couch_btree.5.72034400>,
                                #Fun<couch_db_updater.9.14108461>,1279,2558,
                                true},
                            {btree,<0.792.0>,
                                {2962,<<0,0,0,0,1>>,60},
                                #Fun<couch_db_updater.10.50603258>,
                                #Fun<couch_db_updater.11.85949495>,
                                #Fun<couch_db_updater.6.41937156>,
                                #Fun<couch_db_updater.12.107260449>,1279,2558,
                                true},
                            {btree,<0.792.0>,nil,
                                #Fun<couch_btree.3.59827385>,
                                #Fun<couch_btree.4.7841881>,
                                #Fun<couch_btree.5.72034400>,nil,1279,2558,
                                true},
                            1,<<"_replicator">>,"/data/_replicator.couch.1",
                            [],nil,
                            {user_ctx,null,[],undefined},
                            nil,
                            [before_header,after_header,on_file_open],
                            [create,sys_db,
                             {user_ctx,
                                 {user_ctx,null,
                                     [<<"_admin">>,<<"_replicator">>],
                                     undefined}}]}
** Reason for termination == 
** killed

[error_logger:error,2012-11-06T18:06:58.076,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
  crasher:
    initial call: couch_db:init/1
    pid: <0.796.0>
    registered_name: []
    exception exit: killed
      in function  gen_server:terminate/6
    ancestors: [couch_server,couch_primary_services,couch_server_sup,
                  cb_couch_sup,ns_server_cluster_sup,<0.64.0>]
    messages: []
    links: []
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 987
    stack_size: 24
    reductions: 271
  neighbours:

[error_logger:error,2012-11-06T18:06:58.077,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
  crasher:
    initial call: couch_file:spawn_writer/2
    pid: <0.568.0>
    registered_name: []
    exception exit: {noproc,
                        {gen_server,call,
                            [couch_file_write_guard,
                             {remove,<0.568.0>},
                             infinity]}}
      in function  gen_server:call/3
      in call from couch_file:writer_loop/4
    ancestors: [<0.565.0>,couch_server,couch_primary_services,
                  couch_server_sup,cb_couch_sup,ns_server_cluster_sup,
                  <0.64.0>]
    messages: []
    links: []
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 610
    stack_size: 24
    reductions: 1755
  neighbours:

[error_logger:error,2012-11-06T18:06:58.077,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Generic server <0.569.0> terminating 
** Last message in was {'EXIT',<0.555.0>,killed}
** When Server state == {db,<0.569.0>,<0.570.0>,nil,<<"1352225206524034">>,
                            <0.565.0>,<0.571.0>,
                            {db_header,11,1,
                                <<0,0,0,0,13,103,0,0,0,0,0,51,0,0,0,0,1,0,0,0,
                                  0,0,0,0,0,0,13,69>>,
                                <<0,0,0,0,13,154,0,0,0,0,0,49,0,0,0,0,1>>,
                                nil,0,nil,nil},
                            1,
                            {btree,<0.565.0>,
                                {3431,
                                 <<0,0,0,0,1,0,0,0,0,0,0,0,0,0,13,69>>,
                                 51},
                                #Fun<couch_db_updater.7.89001503>,
                                #Fun<couch_db_updater.8.75953275>,
                                #Fun<couch_btree.5.72034400>,
                                #Fun<couch_db_updater.9.14108461>,1279,2558,
                                true},
                            {btree,<0.565.0>,
                                {3482,<<0,0,0,0,1>>,49},
                                #Fun<couch_db_updater.10.50603258>,
                                #Fun<couch_db_updater.11.85949495>,
                                #Fun<couch_db_updater.6.41937156>,
                                #Fun<couch_db_updater.12.107260449>,1279,2558,
                                true},
                            {btree,<0.565.0>,nil,
                                #Fun<couch_btree.3.59827385>,
                                #Fun<couch_btree.4.7841881>,
                                #Fun<couch_btree.5.72034400>,nil,1279,2558,
                                true},
                            1,<<"_users">>,"/data/_users.couch.1",[],nil,
                            {user_ctx,null,[],undefined},
                            nil,
                            [before_header,after_header,on_file_open],
                            [create,
                             {user_ctx,
                                 {user_ctx,null,[<<"_admin">>],undefined}},
                             sys_db]}
** Reason for termination == 
** killed

[error_logger:error,2012-11-06T18:06:58.078,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
  crasher:
    initial call: couch_db:init/1
    pid: <0.569.0>
    registered_name: []
    exception exit: killed
      in function  gen_server:terminate/6
    ancestors: [couch_server,couch_primary_services,couch_server_sup,
                  cb_couch_sup,ns_server_cluster_sup,<0.64.0>]
    messages: []
    links: []
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 987
    stack_size: 24
    reductions: 260
  neighbours:

[ns_server:error,2012-11-06T18:07:05.519,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:ns_heart<0.408.0>:ns_heart:grab_samples_loading_tasks:319]Failed to grab samples loader tasks: {exit,
                                      {noproc,
                                       {gen_server,call,
                                        [samples_loader_tasks,get_tasks,
                                         2000]}},
                                      [{gen_server,call,3},
                                       {ns_heart,grab_samples_loading_tasks,0},
                                       {ns_heart,current_status,0},
                                       {ns_heart,handle_info,2},
                                       {gen_server,handle_msg,5},
                                       {proc_lib,init_p_do_apply,3}]}
[ns_server:error,2012-11-06T18:07:05.526,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:ns_heart<0.408.0>:ns_heart:grab_samples_loading_tasks:319]Failed to grab samples loader tasks: {exit,
                                      {noproc,
                                       {gen_server,call,
                                        [samples_loader_tasks,get_tasks,
                                         2000]}},
                                      [{gen_server,call,3},
                                       {ns_heart,grab_samples_loading_tasks,0},
                                       {ns_heart,current_status,0},
                                       {ns_heart,handle_call,3},
                                       {gen_server,handle_msg,5},
                                       {proc_lib,init_p_do_apply,3}]}
[couchdb:error,2012-11-06T19:04:45.430,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.269.0>:couch_log:error:42]Uncaught error in HTTP request: {throw,{not_found,no_db_file}}

Stacktrace: [{couch_set_view,open_set_group,2},
             {couch_set_view,get_group_pid,2},
             {couch_set_view,get_group_info,2},
             {couch_set_view_http,route_request,4},
             {couch_httpd,handle_request,6},
             {mochiweb_http,headers,5},
             {proc_lib,init_p_do_apply,3}]
[error_logger:error,2012-11-06T22:29:16.931,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Connection attempt from disallowed node 'ns_1@ec2-50-17-133-35.compute-1.amazonaws.com' ** 

[error_logger:error,2012-11-06T22:29:16.949,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
     Supervisor: {local,ns_bucket_sup}
     Context:    shutdown_error
     Reason:     normal
     Offender:   [{pid,<0.470.0>},
                  {name,buckets_observing_subscription},
                  {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}},
                  {restart_type,permanent},
                  {shutdown,1000},
                  {child_type,worker}]


[error_logger:error,2012-11-06T22:29:17.008,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Connection attempt from disallowed node 'ns_1@ec2-54-242-160-13.compute-1.amazonaws.com' ** 

[error_logger:error,2012-11-06T22:29:17.080,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** Connection attempt from disallowed node 'ns_1@ec2-184-73-53-179.compute-1.amazonaws.com' ** 

[ns_server:error,2012-11-06T22:29:17.806,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:ns_heart<0.11401.1>:ns_heart:grab_samples_loading_tasks:319]Failed to grab samples loader tasks: {exit,
                                      {noproc,
                                       {gen_server,call,
                                        [samples_loader_tasks,get_tasks,
                                         2000]}},
                                      [{gen_server,call,3},
                                       {ns_heart,grab_samples_loading_tasks,0},
                                       {ns_heart,current_status,0},
                                       {ns_heart,handle_info,2},
                                       {gen_server,handle_msg,5},
                                       {proc_lib,init_p_do_apply,3}]}
[ns_server:error,2012-11-06T22:29:17.810,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:ns_heart<0.11401.1>:ns_heart:grab_samples_loading_tasks:319]Failed to grab samples loader tasks: {exit,
                                      {noproc,
                                       {gen_server,call,
                                        [samples_loader_tasks,get_tasks,
                                         2000]}},
                                      [{gen_server,call,3},
                                       {ns_heart,grab_samples_loading_tasks,0},
                                       {ns_heart,current_status,0},
                                       {ns_heart,handle_call,3},
                                       {gen_server,handle_msg,5},
                                       {proc_lib,init_p_do_apply,3}]}
[couchdb:error,2012-11-06T22:29:22.991,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.11662.1>:couch_log:error:42]Set view `default`, main group `_design/A`, writer error
error:      badarith
stacktrace: [{couch_set_view_updater,update_task,1},
             {couch_set_view_updater,flush_writes,1},
             {couch_set_view_updater,do_writes,1},
             {couch_set_view_updater,'-update/7-fun-1-',14}]

[couchdb:error,2012-11-06T22:29:22.992,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.11521.1>:couch_log:error:42]Set view `default`, main group `_design/A`, received error from updater: badarith
[couchdb:error,2012-11-06T22:29:24.845,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.11683.1>:couch_log:error:42]Set view `default`, main group `_design/A`, writer error
error:      badarith
stacktrace: [{couch_set_view_updater,update_task,1},
             {couch_set_view_updater,flush_writes,1},
             {couch_set_view_updater,do_writes,1},
             {couch_set_view_updater,'-update/7-fun-1-',14}]

[couchdb:error,2012-11-06T22:29:24.847,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.11521.1>:couch_log:error:42]Set view `default`, main group `_design/A`, received error from updater: badarith
[ns_server:error,2012-11-06T23:01:37.354,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.11523.1>:ns_memcached:verify_report_long_call:297]call {stats,<<>>} took too long: 821197 us
[ns_server:error,2012-11-07T1:09:36.646,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.11523.1>:ns_memcached:verify_report_long_call:297]call {stats,<<>>} took too long: 854564 us
[ns_server:error,2012-11-07T1:17:59.914,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:ns_memcached-default<0.11518.1>:ns_memcached:handle_info:630]handle_info(ensure_bucket,..) took too long: 627102 us
[error_logger:error,2012-11-07T1:26:10.815,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_msg:76]** gen_event handler {ns_pubsub,#Ref<0.0.4.49245>} crashed.
** Was installed in master_activity_events_ingress
** Last event was: {submit_master_event,
                       {became_master,
                           'ns_1@ec2-174-129-74-54.compute-1.amazonaws.com'}}
** When handler state == {state,#Fun<master_activity_events.2.47149335>,ok}
** Reason == {timeout,{gen_fsm,sync_send_all_state_event,
                               [mb_master,master_node]}}

[error_logger:error,2012-11-07T1:26:10.850,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
  crasher:
    initial call: ns_pubsub:do_subscribe_link/4
    pid: <0.11426.1>
    registered_name: []
    exception exit: {handler_crashed,master_activity_events_ingress,
                        {'EXIT',
                            {timeout,
                                {gen_fsm,sync_send_all_state_event,
                                    [mb_master,master_node]}}}}
      in function  ns_pubsub:do_subscribe_link/4
    ancestors: [ns_server_sup,ns_server_cluster_sup,<0.64.0>]
    messages: []
    links: [<0.11369.1>,<0.11425.1>]
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 233
    stack_size: 24
    reductions: 115
  neighbours:

[error_logger:error,2012-11-07T1:26:10.851,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
     Supervisor: {local,ns_server_sup}
     Context:    child_terminated
     Reason:     {handler_crashed,master_activity_events_ingress,
                     {'EXIT',
                         {timeout,
                             {gen_fsm,sync_send_all_state_event,
                                 [mb_master,master_node]}}}}
     Offender:   [{pid,<0.11426.1>},
                  {name,master_activity_events_timestamper},
                  {mfargs,{master_activity_events,start_link_timestamper,[]}},
                  {restart_type,permanent},
                  {shutdown,brutal_kill},
                  {child_type,worker}]


[error_logger:error,2012-11-07T1:26:46.523,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
     Supervisor: {local,mb_master_sup}
     Context:    child_terminated
     Reason:     normal
     Offender:   [{pid,<0.30106.19>},
                  {name,ns_tick},
                  {mfargs,{ns_tick,start_link,[]}},
                  {restart_type,permanent},
                  {shutdown,10},
                  {child_type,worker}]


[error_logger:error,2012-11-07T1:26:47.400,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
     Supervisor: {local,mb_master_sup}
     Context:    child_terminated
     Reason:     normal
     Offender:   [{pid,<0.30108.19>},
                  {name,auto_failover},
                  {mfargs,{auto_failover,start_link,[]}},
                  {restart_type,permanent},
                  {shutdown,10},
                  {child_type,worker}]


[ns_server:error,2012-11-07T1:26:47.927,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:ns_doctor<0.11404.1>:ns_doctor:update_status:205]The following buckets became not ready on node 'ns_1@ec2-184-73-53-179.compute-1.amazonaws.com': ["default"], those of them are active ["default"]
[error_logger:error,2012-11-07T1:26:47.931,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:error_logger<0.5.0>:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
     Supervisor: {local,mb_master_sup}
     Context:    child_terminated
     Reason:     normal
     Offender:   [{pid,<0.30104.19>},
                  {name,ns_orchestrator},
                  {mfargs,{ns_orchestrator,start_link,[]}},
                  {restart_type,permanent},
                  {shutdown,20},
                  {child_type,worker}]


[ns_server:error,2012-11-07T1:26:53.735,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.30159.19>:ns_janitor:cleanup_with_states:84]Bucket "default" not yet ready on ['ns_1@ec2-184-73-53-179.compute-1.amazonaws.com']
[ns_server:error,2012-11-07T1:27:03.689,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.30312.19>:ns_janitor:cleanup_with_states:84]Bucket "default" not yet ready on ['ns_1@ec2-184-73-53-179.compute-1.amazonaws.com']
[ns_server:error,2012-11-07T1:27:13.690,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.30375.19>:ns_janitor:cleanup_with_states:84]Bucket "default" not yet ready on ['ns_1@ec2-184-73-53-179.compute-1.amazonaws.com']
[ns_server:error,2012-11-07T1:27:23.690,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.30450.19>:ns_janitor:cleanup_with_states:84]Bucket "default" not yet ready on ['ns_1@ec2-184-73-53-179.compute-1.amazonaws.com']
[ns_server:error,2012-11-07T1:27:33.689,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.30505.19>:ns_janitor:cleanup_with_states:84]Bucket "default" not yet ready on ['ns_1@ec2-184-73-53-179.compute-1.amazonaws.com']
[ns_server:error,2012-11-07T1:27:43.692,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.30567.19>:ns_janitor:cleanup_with_states:84]Bucket "default" not yet ready on ['ns_1@ec2-184-73-53-179.compute-1.amazonaws.com']
[ns_server:error,2012-11-07T1:27:53.689,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.30632.19>:ns_janitor:cleanup_with_states:84]Bucket "default" not yet ready on ['ns_1@ec2-184-73-53-179.compute-1.amazonaws.com']
[ns_server:error,2012-11-07T1:28:03.689,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.30685.19>:ns_janitor:cleanup_with_states:84]Bucket "default" not yet ready on ['ns_1@ec2-184-73-53-179.compute-1.amazonaws.com']
[ns_server:error,2012-11-07T1:28:13.689,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.30769.19>:ns_janitor:cleanup_with_states:84]Bucket "default" not yet ready on ['ns_1@ec2-184-73-53-179.compute-1.amazonaws.com']
[ns_server:error,2012-11-07T1:37:26.117,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:ns_doctor<0.11404.1>:ns_doctor:update_status:205]The following buckets became not ready on node 'ns_1@ec2-184-73-53-179.compute-1.amazonaws.com': ["default"], those of them are active ["default"]
[ns_server:error,2012-11-07T1:44:39.844,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:ns_memcached-default<0.11518.1>:ns_memcached:handle_info:630]handle_info(ensure_bucket,..) took too long: 552653 us
[ns_server:error,2012-11-07T2:02:33.689,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.5071.24>:ns_janitor:cleanup_with_states:84]Bucket "default" not yet ready on ['ns_1@ec2-50-17-133-35.compute-1.amazonaws.com']
[ns_server:error,2012-11-07T4:00:48.602,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.11523.1>:ns_memcached:verify_report_long_call:297]call topkeys took too long: 784904 us
[ns_server:error,2012-11-07T4:02:13.696,ns_1@ec2-174-129-74-54.compute-1.amazonaws.com:<0.11525.1>:ns_memcached:verify_report_long_call:297]call topkeys took too long: 878691 us
