[2019-12-04 11:16:21,493] - [eventing_base] [140432460064512] - INFO - Starting Test: test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutations [2019-12-04 11:16:21,543] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.100 with username:root [2019-12-04 11:16:21,712] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.100 [2019-12-04 11:16:22,567] - [rest_client] [140432460064512] - INFO - Node version in cluster 6.5.0-4917-enterprise [2019-12-04 11:16:22,611] - [rest_client] [140432460064512] - ERROR - GET http://172.23.104.100:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 404 reason: unknown "unknown pool" auth: Administrator:password [2019-12-04 11:16:22,612] - [rest_client] [140432460064512] - INFO - Node versions in cluster [] [2019-12-04 11:16:22,613] - [basetestcase] [140432460064512] - INFO - ============== basetestcase setup was started for test #13 test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutations============== [2019-12-04 11:16:22,640] - [rest_client] [140432460064512] - ERROR - GET http://172.23.104.100:8091/pools/default/ body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 404 reason: unknown "unknown pool" auth: Administrator:password [2019-12-04 11:16:23,127] - [cluster_helper] [140432460064512] - INFO - waiting for ns_server @ 172.23.104.100:8091 [2019-12-04 11:16:23,141] - [cluster_helper] [140432460064512] - INFO - ns_server @ 172.23.104.100:8091 is running [2019-12-04 11:16:23,160] - [cluster_helper] [140432460064512] - INFO - waiting for ns_server @ 172.23.104.103:8091 [2019-12-04 11:16:23,177] - [cluster_helper] [140432460064512] - INFO - ns_server @ 172.23.104.103:8091 is running [2019-12-04 11:16:23,214] - [cluster_helper] [140432460064512] - INFO - waiting for ns_server @ 172.23.104.105:8091 [2019-12-04 11:16:23,231] - [cluster_helper] [140432460064512] - INFO - ns_server @ 172.23.104.105:8091 is running [2019-12-04 11:16:23,246] - [cluster_helper] [140432460064512] - INFO - waiting for ns_server @ 172.23.104.107:8091 [2019-12-04 11:16:23,261] - [cluster_helper] [140432460064512] - INFO - ns_server @ 172.23.104.107:8091 is running [2019-12-04 11:16:23,276] - [cluster_helper] [140432460064512] - INFO - waiting for ns_server @ 172.23.104.109:8091 [2019-12-04 11:16:23,311] - [cluster_helper] [140432460064512] - INFO - ns_server @ 172.23.104.109:8091 is running [2019-12-04 11:16:23,324] - [cluster_helper] [140432460064512] - INFO - waiting for ns_server @ 172.23.104.199:8091 [2019-12-04 11:16:23,338] - [cluster_helper] [140432460064512] - INFO - ns_server @ 172.23.104.199:8091 is running [2019-12-04 11:16:23,353] - [cluster_helper] [140432460064512] - INFO - waiting for ns_server @ 172.23.104.201:8091 [2019-12-04 11:16:23,366] - [cluster_helper] [140432460064512] - INFO - ns_server @ 172.23.104.201:8091 is running [2019-12-04 11:16:23,382] - [cluster_helper] [140432460064512] - INFO - waiting for ns_server @ 172.23.104.202:8091 [2019-12-04 11:16:23,414] - [cluster_helper] [140432460064512] - INFO - ns_server @ 172.23.104.202:8091 is running [2019-12-04 11:16:23,415] - [basetestcase] [140432460064512] - INFO - initializing cluster [2019-12-04 11:16:23,419] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.100 with username:root [2019-12-04 11:16:23,591] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.100 [2019-12-04 11:16:24,440] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.100 with username:root [2019-12-04 11:16:24,612] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.100 [2019-12-04 11:16:25,522] - [remote_util] [140432460064512] - INFO - 172.23.104.100 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:16:25,721] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:16:25,722] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.100: systemctl stop couchbase-server.service [2019-12-04 11:16:27,849] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:27,849] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:16:27,964] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.100: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:16:28,001] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:28,002] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.100: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:16:28,076] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:28,080] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.100 with username:root [2019-12-04 11:16:28,249] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.100 [2019-12-04 11:16:29,139] - [remote_util] [140432460064512] - INFO - 172.23.104.100 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:16:29,334] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:16:29,335] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.100: systemctl start couchbase-server.service [2019-12-04 11:16:29,372] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:29,373] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:16:29,491] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.103 with username:root [2019-12-04 11:16:29,659] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.103 [2019-12-04 11:16:30,508] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.103 with username:root [2019-12-04 11:16:30,680] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.103 [2019-12-04 11:16:31,568] - [remote_util] [140432460064512] - INFO - 172.23.104.103 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:16:31,784] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:16:31,785] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.103: systemctl stop couchbase-server.service [2019-12-04 11:16:33,414] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:33,415] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:16:33,531] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.103: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:16:33,560] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:33,560] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.103: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:16:33,645] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:33,652] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.103 with username:root [2019-12-04 11:16:33,788] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.103 [2019-12-04 11:16:34,903] - [remote_util] [140432460064512] - INFO - 172.23.104.103 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:16:35,165] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:16:35,165] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.103: systemctl start couchbase-server.service [2019-12-04 11:16:35,202] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:35,203] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:16:35,329] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.105 with username:root [2019-12-04 11:16:35,434] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.105 [2019-12-04 11:16:36,309] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.105 with username:root [2019-12-04 11:16:36,477] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.105 [2019-12-04 11:16:37,371] - [remote_util] [140432460064512] - INFO - 172.23.104.105 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:16:37,654] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:16:37,657] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.105: systemctl stop couchbase-server.service [2019-12-04 11:16:39,235] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:39,236] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:16:39,352] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.105: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:16:39,375] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:39,376] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.105: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:16:39,459] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:39,471] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.105 with username:root [2019-12-04 11:16:39,572] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.105 [2019-12-04 11:16:40,503] - [remote_util] [140432460064512] - INFO - 172.23.104.105 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:16:40,646] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:16:40,648] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.105: systemctl start couchbase-server.service [2019-12-04 11:16:40,675] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:40,676] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:16:40,794] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.107 with username:root [2019-12-04 11:16:40,963] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.107 [2019-12-04 11:16:42,581] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.107 with username:root [2019-12-04 11:16:42,751] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.107 [2019-12-04 11:16:44,359] - [remote_util] [140432460064512] - INFO - 172.23.104.107 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:16:44,720] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:16:44,721] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.107: systemctl stop couchbase-server.service [2019-12-04 11:16:46,747] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:46,748] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:16:46,863] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.107: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:16:46,938] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:46,939] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.107: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:16:47,072] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:47,084] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.107 with username:root [2019-12-04 11:16:47,256] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.107 [2019-12-04 11:16:48,839] - [remote_util] [140432460064512] - INFO - 172.23.104.107 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:16:49,206] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:16:49,207] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.107: systemctl start couchbase-server.service [2019-12-04 11:16:49,328] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:49,329] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:16:49,452] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.109 with username:root [2019-12-04 11:16:49,624] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.109 [2019-12-04 11:16:51,121] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.109 with username:root [2019-12-04 11:16:51,292] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.109 [2019-12-04 11:16:52,813] - [remote_util] [140432460064512] - INFO - 172.23.104.109 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:16:53,158] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:16:53,159] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.109: systemctl stop couchbase-server.service [2019-12-04 11:16:55,188] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:55,190] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:16:55,255] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.109: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:16:55,327] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:55,328] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.109: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:16:55,463] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:55,470] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.109 with username:root [2019-12-04 11:16:55,638] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.109 [2019-12-04 11:16:57,311] - [remote_util] [140432460064512] - INFO - 172.23.104.109 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:16:57,619] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:16:57,620] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.109: systemctl start couchbase-server.service [2019-12-04 11:16:57,741] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:16:57,742] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:16:57,861] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.199 with username:root [2019-12-04 11:16:58,031] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.199 [2019-12-04 11:16:58,999] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.199 with username:root [2019-12-04 11:16:59,168] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.199 [2019-12-04 11:17:00,240] - [remote_util] [140432460064512] - INFO - 172.23.104.199 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:17:00,486] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:17:00,488] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.199: systemctl stop couchbase-server.service [2019-12-04 11:17:02,466] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:17:02,466] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:17:02,581] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.199: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:17:02,621] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:17:02,622] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.199: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:17:02,694] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:17:02,699] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.199 with username:root [2019-12-04 11:17:02,868] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.199 [2019-12-04 11:17:04,007] - [remote_util] [140432460064512] - INFO - 172.23.104.199 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:17:04,341] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:17:04,342] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.199: systemctl start couchbase-server.service [2019-12-04 11:17:04,414] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:17:04,414] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:17:04,542] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.201 with username:root [2019-12-04 11:17:04,711] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.201 [2019-12-04 11:17:05,664] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.201 with username:root [2019-12-04 11:17:05,833] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.201 [2019-12-04 11:17:06,839] - [remote_util] [140432460064512] - INFO - 172.23.104.201 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:17:07,038] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:17:07,039] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.201: systemctl stop couchbase-server.service [2019-12-04 11:17:09,020] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:17:09,021] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:17:09,137] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.201: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:17:09,183] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:17:09,183] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.201: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:17:09,252] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:17:09,261] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.201 with username:root [2019-12-04 11:17:09,431] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.201 [2019-12-04 11:17:10,466] - [remote_util] [140432460064512] - INFO - 172.23.104.201 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:17:10,704] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:17:10,705] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.201: systemctl start couchbase-server.service [2019-12-04 11:17:10,777] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:17:10,777] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:17:10,896] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.202 with username:root [2019-12-04 11:17:11,065] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.202 [2019-12-04 11:17:12,089] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.202 with username:root [2019-12-04 11:17:12,258] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.202 [2019-12-04 11:17:13,427] - [remote_util] [140432460064512] - INFO - 172.23.104.202 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:17:13,707] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:17:13,707] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.202: systemctl stop couchbase-server.service [2019-12-04 11:17:15,886] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:17:15,886] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:17:16,001] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.202: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:17:16,040] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:17:16,041] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.202: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:17:16,109] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:17:16,114] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.202 with username:root [2019-12-04 11:17:16,282] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.202 [2019-12-04 11:17:17,432] - [remote_util] [140432460064512] - INFO - 172.23.104.202 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:17:17,631] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:17:17,631] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.202: systemctl start couchbase-server.service [2019-12-04 11:17:17,703] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:17:17,704] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:17:17,819] - [basetestcase] [140432460064512] - INFO - sleep for 10 secs. ... [2019-12-04 11:17:28,636] - [task] [140424073172736] - INFO - server: ip:172.23.104.100 port:8091 ssh_username:root, nodes/self: {'ip': u'172.23.104.100', 'availableStorage': [], 'rest_username': '', 'id': u'ns_1@172.23.104.100', 'uptime': u'52', 'mcdMemoryReserved': 3349, 'storageTotalRam': 4187, 'hostname': u'172.23.104.100:8091', 'storage': [], 'moxi': 11211, 'port': u'8091', 'version': u'6.5.0-4917-enterprise', 'memcached': 11210, 'status': u'healthy', 'clusterCompatibility': 393221, 'curr_items': 0, 'services': [u'kv'], 'rest_password': '', 'clusterMembership': u'active', 'memoryFree': 3438460928, 'memoryTotal': 4390723584, 'memoryQuota': 1846, 'mcdMemoryAllocated': 3349, 'os': u'x86_64-unknown-linux-gnu', 'ports': []} [2019-12-04 11:17:28,653] - [rest_client] [140424073172736] - ERROR - GET http://172.23.104.100:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 404 reason: unknown "unknown pool" auth: Administrator:password [2019-12-04 11:17:28,656] - [rest_client] [140424073172736] - INFO - pools/default params : memoryQuota=2243 [2019-12-04 11:17:28,677] - [rest_client] [140424073172736] - INFO - /node/controller/setupServices params on 172.23.104.100:8091:services=kv&password=password&user=Administrator&hostname=172.23.104.100 [2019-12-04 11:17:28,704] - [rest_client] [140424073172736] - INFO - settings/web params on 172.23.104.100:8091:username=Administrator&password=password&port=8091 [2019-12-04 11:17:28,813] - [remote_util] [140424073172736] - INFO - connecting to 172.23.104.100 with username:root [2019-12-04 11:17:28,951] - [remote_util] [140424073172736] - INFO - Connected to 172.23.104.100 [2019-12-04 11:17:29,840] - [remote_util] [140424073172736] - INFO - running command.raw on 172.23.104.100: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' [2019-12-04 11:17:29,917] - [remote_util] [140424073172736] - INFO - command executed successfully [2019-12-04 11:17:29,929] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.100:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:29,942] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.100:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:29,942] - [rest_client] [140424073172736] - INFO - settings/indexes params : storageMode=plasma [2019-12-04 11:17:30,055] - [task] [140424073172736] - INFO - server: ip:172.23.104.103 port:8091 ssh_username:root, nodes/self: {'ip': u'172.23.104.103', 'availableStorage': [], 'rest_username': '', 'id': u'ns_1@172.23.104.103', 'uptime': u'48', 'mcdMemoryReserved': 3219, 'storageTotalRam': 4024, 'hostname': u'172.23.104.103:8091', 'storage': [], 'moxi': 11211, 'port': u'8091', 'version': u'6.5.0-4917-enterprise', 'memcached': 11210, 'status': u'healthy', 'clusterCompatibility': 393221, 'curr_items': 0, 'services': [u'kv'], 'rest_password': '', 'clusterMembership': u'active', 'memoryFree': 3634057216, 'memoryTotal': 4220149760, 'memoryQuota': 1748, 'mcdMemoryAllocated': 3219, 'os': u'x86_64-unknown-linux-gnu', 'ports': []} [2019-12-04 11:17:30,072] - [rest_client] [140424073172736] - ERROR - GET http://172.23.104.103:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 404 reason: unknown "unknown pool" auth: Administrator:password [2019-12-04 11:17:30,073] - [rest_client] [140424073172736] - INFO - pools/default params : memoryQuota=2156 [2019-12-04 11:17:30,110] - [rest_client] [140424073172736] - INFO - settings/web params on 172.23.104.103:8091:username=Administrator&password=password&port=8091 [2019-12-04 11:17:30,188] - [remote_util] [140424073172736] - INFO - connecting to 172.23.104.103 with username:root [2019-12-04 11:17:30,295] - [remote_util] [140424073172736] - INFO - Connected to 172.23.104.103 [2019-12-04 11:17:31,220] - [remote_util] [140424073172736] - INFO - running command.raw on 172.23.104.103: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' [2019-12-04 11:17:31,257] - [remote_util] [140424073172736] - INFO - command executed successfully [2019-12-04 11:17:31,276] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.103:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:31,301] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.103:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:31,305] - [rest_client] [140424073172736] - INFO - settings/indexes params : storageMode=plasma [2019-12-04 11:17:31,432] - [task] [140424073172736] - INFO - server: ip:172.23.104.105 port:8091 ssh_username:root, nodes/self: {'ip': u'172.23.104.105', 'availableStorage': [], 'rest_username': '', 'id': u'ns_1@172.23.104.105', 'uptime': u'48', 'mcdMemoryReserved': 3220, 'storageTotalRam': 4025, 'hostname': u'172.23.104.105:8091', 'storage': [], 'moxi': 11211, 'port': u'8091', 'version': u'6.5.0-4917-enterprise', 'memcached': 11210, 'status': u'healthy', 'clusterCompatibility': 393221, 'curr_items': 0, 'services': [u'kv'], 'rest_password': '', 'clusterMembership': u'active', 'memoryFree': 3663200256, 'memoryTotal': 4220854272, 'memoryQuota': 1749, 'mcdMemoryAllocated': 3220, 'os': u'x86_64-unknown-linux-gnu', 'ports': []} [2019-12-04 11:17:31,447] - [rest_client] [140424073172736] - ERROR - GET http://172.23.104.105:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 404 reason: unknown "unknown pool" auth: Administrator:password [2019-12-04 11:17:31,448] - [rest_client] [140424073172736] - INFO - pools/default params : memoryQuota=2157 [2019-12-04 11:17:31,475] - [rest_client] [140424073172736] - INFO - settings/web params on 172.23.104.105:8091:username=Administrator&password=password&port=8091 [2019-12-04 11:17:31,549] - [remote_util] [140424073172736] - INFO - connecting to 172.23.104.105 with username:root [2019-12-04 11:17:31,684] - [remote_util] [140424073172736] - INFO - Connected to 172.23.104.105 [2019-12-04 11:17:32,532] - [remote_util] [140424073172736] - INFO - running command.raw on 172.23.104.105: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' [2019-12-04 11:17:32,570] - [remote_util] [140424073172736] - INFO - command executed successfully [2019-12-04 11:17:32,585] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.105:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:32,612] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.105:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:32,612] - [rest_client] [140424073172736] - INFO - settings/indexes params : storageMode=plasma [2019-12-04 11:17:32,705] - [task] [140424073172736] - INFO - server: ip:172.23.104.107 port:8091 ssh_username:root, nodes/self: {'ip': u'172.23.104.107', 'availableStorage': [], 'rest_username': '', 'id': u'ns_1@172.23.104.107', 'uptime': u'36', 'mcdMemoryReserved': 3220, 'storageTotalRam': 4025, 'hostname': u'172.23.104.107:8091', 'storage': [], 'moxi': 11211, 'port': u'8091', 'version': u'6.5.0-4917-enterprise', 'memcached': 11210, 'status': u'healthy', 'clusterCompatibility': 393221, 'curr_items': 0, 'services': [u'kv'], 'rest_password': '', 'clusterMembership': u'active', 'memoryFree': 3670589440, 'memoryTotal': 4220854272, 'memoryQuota': 1749, 'mcdMemoryAllocated': 3220, 'os': u'x86_64-unknown-linux-gnu', 'ports': []} [2019-12-04 11:17:32,725] - [rest_client] [140424073172736] - ERROR - GET http://172.23.104.107:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 404 reason: unknown "unknown pool" auth: Administrator:password [2019-12-04 11:17:32,726] - [rest_client] [140424073172736] - INFO - pools/default params : memoryQuota=2157 [2019-12-04 11:17:32,745] - [rest_client] [140424073172736] - INFO - settings/web params on 172.23.104.107:8091:username=Administrator&password=password&port=8091 [2019-12-04 11:17:32,848] - [remote_util] [140424073172736] - INFO - connecting to 172.23.104.107 with username:root [2019-12-04 11:17:33,016] - [remote_util] [140424073172736] - INFO - Connected to 172.23.104.107 [2019-12-04 11:17:34,609] - [remote_util] [140424073172736] - INFO - running command.raw on 172.23.104.107: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' [2019-12-04 11:17:34,731] - [remote_util] [140424073172736] - INFO - command executed successfully [2019-12-04 11:17:34,748] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.107:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:34,773] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.107:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:34,775] - [rest_client] [140424073172736] - INFO - settings/indexes params : storageMode=plasma [2019-12-04 11:17:34,879] - [task] [140424073172736] - INFO - server: ip:172.23.104.109 port:8091 ssh_username:root, nodes/self: {'ip': u'172.23.104.109', 'availableStorage': [], 'rest_username': '', 'id': u'ns_1@172.23.104.109', 'uptime': u'30', 'mcdMemoryReserved': 3220, 'storageTotalRam': 4025, 'hostname': u'172.23.104.109:8091', 'storage': [], 'moxi': 11211, 'port': u'8091', 'version': u'6.5.0-4917-enterprise', 'memcached': 11210, 'status': u'healthy', 'clusterCompatibility': 393221, 'curr_items': 0, 'services': [u'kv'], 'rest_password': '', 'clusterMembership': u'active', 'memoryFree': 3686785024, 'memoryTotal': 4220854272, 'memoryQuota': 1749, 'mcdMemoryAllocated': 3220, 'os': u'x86_64-unknown-linux-gnu', 'ports': []} [2019-12-04 11:17:34,896] - [rest_client] [140424073172736] - ERROR - GET http://172.23.104.109:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 404 reason: unknown "unknown pool" auth: Administrator:password [2019-12-04 11:17:34,897] - [rest_client] [140424073172736] - INFO - pools/default params : memoryQuota=2157 [2019-12-04 11:17:34,923] - [rest_client] [140424073172736] - INFO - settings/web params on 172.23.104.109:8091:username=Administrator&password=password&port=8091 [2019-12-04 11:17:35,035] - [remote_util] [140424073172736] - INFO - connecting to 172.23.104.109 with username:root [2019-12-04 11:17:35,204] - [remote_util] [140424073172736] - INFO - Connected to 172.23.104.109 [2019-12-04 11:17:36,697] - [remote_util] [140424073172736] - INFO - running command.raw on 172.23.104.109: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' [2019-12-04 11:17:36,820] - [remote_util] [140424073172736] - INFO - command executed successfully [2019-12-04 11:17:36,844] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.109:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:36,857] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.109:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:36,858] - [rest_client] [140424073172736] - INFO - settings/indexes params : storageMode=plasma [2019-12-04 11:17:36,958] - [task] [140424073172736] - INFO - server: ip:172.23.104.199 port:8091 ssh_username:root, nodes/self: {'ip': u'172.23.104.199', 'availableStorage': [], 'rest_username': '', 'id': u'ns_1@172.23.104.199', 'uptime': u'28', 'mcdMemoryReserved': 3205, 'storageTotalRam': 4007, 'hostname': u'172.23.104.199:8091', 'storage': [], 'moxi': 11211, 'port': u'8091', 'version': u'6.5.0-4917-enterprise', 'memcached': 11210, 'status': u'healthy', 'clusterCompatibility': 393221, 'curr_items': 0, 'services': [u'kv'], 'rest_password': '', 'clusterMembership': u'active', 'memoryFree': 3576868864, 'memoryTotal': 4201844736, 'memoryQuota': 1738, 'mcdMemoryAllocated': 3205, 'os': u'x86_64-unknown-linux-gnu', 'ports': []} [2019-12-04 11:17:36,971] - [rest_client] [140424073172736] - ERROR - GET http://172.23.104.199:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 404 reason: unknown "unknown pool" auth: Administrator:password [2019-12-04 11:17:36,972] - [rest_client] [140424073172736] - INFO - pools/default params : memoryQuota=2147 [2019-12-04 11:17:36,993] - [rest_client] [140424073172736] - INFO - settings/web params on 172.23.104.199:8091:username=Administrator&password=password&port=8091 [2019-12-04 11:17:37,150] - [remote_util] [140424073172736] - INFO - connecting to 172.23.104.199 with username:root [2019-12-04 11:17:37,319] - [remote_util] [140424073172736] - INFO - Connected to 172.23.104.199 [2019-12-04 11:17:38,388] - [remote_util] [140424073172736] - INFO - running command.raw on 172.23.104.199: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' [2019-12-04 11:17:38,459] - [remote_util] [140424073172736] - INFO - command executed successfully [2019-12-04 11:17:38,472] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.199:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:38,490] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.199:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:38,491] - [rest_client] [140424073172736] - INFO - settings/indexes params : storageMode=plasma [2019-12-04 11:17:38,597] - [task] [140424073172736] - INFO - server: ip:172.23.104.201 port:8091 ssh_username:root, nodes/self: {'ip': u'172.23.104.201', 'availableStorage': [], 'rest_username': '', 'id': u'ns_1@172.23.104.201', 'uptime': u'21', 'mcdMemoryReserved': 3205, 'storageTotalRam': 4007, 'hostname': u'172.23.104.201:8091', 'storage': [], 'moxi': 11211, 'port': u'8091', 'version': u'6.5.0-4917-enterprise', 'memcached': 11210, 'status': u'healthy', 'clusterCompatibility': 393221, 'curr_items': 0, 'services': [u'kv'], 'rest_password': '', 'clusterMembership': u'active', 'memoryFree': 3573354496, 'memoryTotal': 4201848832, 'memoryQuota': 1738, 'mcdMemoryAllocated': 3205, 'os': u'x86_64-unknown-linux-gnu', 'ports': []} [2019-12-04 11:17:38,622] - [rest_client] [140424073172736] - ERROR - GET http://172.23.104.201:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 404 reason: unknown "unknown pool" auth: Administrator:password [2019-12-04 11:17:38,624] - [rest_client] [140424073172736] - INFO - pools/default params : memoryQuota=2147 [2019-12-04 11:17:38,645] - [rest_client] [140424073172736] - INFO - settings/web params on 172.23.104.201:8091:username=Administrator&password=password&port=8091 [2019-12-04 11:17:38,770] - [remote_util] [140424073172736] - INFO - connecting to 172.23.104.201 with username:root [2019-12-04 11:17:38,941] - [remote_util] [140424073172736] - INFO - Connected to 172.23.104.201 [2019-12-04 11:17:39,867] - [remote_util] [140424073172736] - INFO - running command.raw on 172.23.104.201: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' [2019-12-04 11:17:39,936] - [remote_util] [140424073172736] - INFO - command executed successfully [2019-12-04 11:17:39,953] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.201:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:39,965] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.201:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:39,966] - [rest_client] [140424073172736] - INFO - settings/indexes params : storageMode=plasma [2019-12-04 11:17:40,068] - [task] [140424073172736] - INFO - server: ip:172.23.104.202 port:8091 ssh_username:root, nodes/self: {'ip': u'172.23.104.202', 'availableStorage': [], 'rest_username': '', 'id': u'ns_1@cb.local', 'uptime': u'17', 'mcdMemoryReserved': 3205, 'storageTotalRam': 4007, 'hostname': u'172.23.104.202:8091', 'storage': [], 'moxi': 11211, 'port': u'8091', 'version': u'6.5.0-4917-enterprise', 'memcached': 11210, 'status': u'healthy', 'clusterCompatibility': 393221, 'curr_items': 0, 'services': [u'kv'], 'rest_password': '', 'clusterMembership': u'active', 'memoryFree': 3563626496, 'memoryTotal': 4201844736, 'memoryQuota': 1738, 'mcdMemoryAllocated': 3205, 'os': u'x86_64-unknown-linux-gnu', 'ports': []} [2019-12-04 11:17:40,081] - [rest_client] [140424073172736] - ERROR - GET http://172.23.104.202:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 404 reason: unknown "unknown pool" auth: Administrator:password [2019-12-04 11:17:40,083] - [rest_client] [140424073172736] - INFO - pools/default params : memoryQuota=2147 [2019-12-04 11:17:40,104] - [rest_client] [140424073172736] - INFO - settings/web params on 172.23.104.202:8091:username=Administrator&password=password&port=8091 [2019-12-04 11:17:40,248] - [remote_util] [140424073172736] - INFO - connecting to 172.23.104.202 with username:root [2019-12-04 11:17:40,416] - [remote_util] [140424073172736] - INFO - Connected to 172.23.104.202 [2019-12-04 11:17:41,550] - [remote_util] [140424073172736] - INFO - running command.raw on 172.23.104.202: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' [2019-12-04 11:17:41,620] - [remote_util] [140424073172736] - INFO - command executed successfully [2019-12-04 11:17:41,649] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.202:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:41,663] - [rest_client] [140424073172736] - INFO - /diag/eval status on 172.23.104.202:8091: True content: [6,5] command: cluster_compat_mode:get_compat_version(). [2019-12-04 11:17:41,664] - [rest_client] [140424073172736] - INFO - settings/indexes params : storageMode=plasma [2019-12-04 11:17:41,772] - [basetestcase] [140432460064512] - INFO - **** add built-in 'cbadminbucket' user to node 172.23.104.100 **** [2019-12-04 11:17:41,854] - [rest_client] [140432460064512] - ERROR - DELETE http://172.23.104.100:8091/settings/rbac/users/local/cbadminbucket body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 404 reason: unknown "User was not found." auth: Administrator:password [2019-12-04 11:17:41,856] - [internal_user] [140432460064512] - INFO - Exception while deleting user. Exception is -"User was not found." [2019-12-04 11:17:42,009] - [basetestcase] [140432460064512] - INFO - sleep for 5 secs. ... [2019-12-04 11:17:47,011] - [basetestcase] [140432460064512] - INFO - **** add 'admin' role to 'cbadminbucket' user **** [2019-12-04 11:17:47,076] - [basetestcase] [140432460064512] - INFO - done initializing cluster [2019-12-04 11:17:47,741] - [task] [140424073172736] - INFO - adding node 172.23.104.103:8091 to cluster [2019-12-04 11:17:47,742] - [rest_client] [140424073172736] - INFO - adding remote node @172.23.104.103:8091 to this cluster @172.23.104.100:8091 [2019-12-04 11:17:50,086] - [task] [140424073172736] - INFO - adding node 172.23.104.105:8091 to cluster [2019-12-04 11:17:50,087] - [rest_client] [140424073172736] - INFO - adding remote node @172.23.104.105:8091 to this cluster @172.23.104.100:8091 [2019-12-04 11:17:52,035] - [task] [140424073172736] - INFO - adding node 172.23.104.107:8091 to cluster [2019-12-04 11:17:52,036] - [rest_client] [140424073172736] - INFO - adding remote node @172.23.104.107:8091 to this cluster @172.23.104.100:8091 [2019-12-04 11:17:55,689] - [task] [140424073172736] - INFO - adding node 172.23.104.109:8091 to cluster [2019-12-04 11:17:55,689] - [rest_client] [140424073172736] - INFO - adding remote node @172.23.104.109:8091 to this cluster @172.23.104.100:8091 [2019-12-04 11:17:59,353] - [task] [140424073172736] - INFO - adding node 172.23.104.199:8091 to cluster [2019-12-04 11:17:59,355] - [rest_client] [140424073172736] - INFO - adding remote node @172.23.104.199:8091 to this cluster @172.23.104.100:8091 [2019-12-04 11:18:03,851] - [task] [140424073172736] - INFO - adding node 172.23.104.201:8091 to cluster [2019-12-04 11:18:03,852] - [rest_client] [140424073172736] - INFO - adding remote node @172.23.104.201:8091 to this cluster @172.23.104.100:8091 [2019-12-04 11:18:07,732] - [rest_client] [140424073172736] - INFO - rebalance params : {'password': 'password', 'ejectedNodes': '', 'user': 'Administrator', 'knownNodes': u'ns_1@172.23.104.107,ns_1@172.23.104.100,ns_1@172.23.104.199,ns_1@172.23.104.109,ns_1@172.23.104.201,ns_1@172.23.104.105,ns_1@172.23.104.103'} [2019-12-04 11:18:07,758] - [rest_client] [140424073172736] - INFO - rebalance operation started [2019-12-04 11:18:07,772] - [rest_client] [140424073172736] - INFO - rebalance percentage : 0.00 % [2019-12-04 11:18:07,773] - [task] [140424073172736] - INFO - Rebalance - status: running, progress: 0.00% [2019-12-04 11:18:17,874] - [task] [140424073172736] - INFO - Rebalance - status: none, progress: 100.00% [2019-12-04 11:18:17,904] - [task] [140424073172736] - INFO - rebalancing was completed with progress: 100% in 10.1452419758 sec [2019-12-04 11:18:17,916] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.100 with username:root [2019-12-04 11:18:18,088] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.100 [2019-12-04 11:18:19,048] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.100: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' [2019-12-04 11:18:19,118] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:18:19,119] - [basetestcase] [140432460064512] - INFO - Enabled diag/eval for non-local hosts from 172.23.104.100 [2019-12-04 11:18:19,121] - [basetestcase] [140432460064512] - INFO - ============== basetestcase setup was finished for test #13 test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutations ============== [2019-12-04 11:18:19,207] - [basetestcase] [140432460064512] - INFO - ------- Cluster statistics ------- [2019-12-04 11:18:19,208] - [basetestcase] [140432460064512] - INFO - 172.23.104.107:8091 => {'swap_mem_used': 32768, 'cpu_utilization': 1.503759398496241, 'mem_free': 3599200256, 'swap_mem_total': 3758092288, 'mem_total': 4220854272, 'services': [u'eventing']} [2019-12-04 11:18:19,208] - [basetestcase] [140432460064512] - INFO - 172.23.104.100:8091 => {'swap_mem_used': 0, 'cpu_utilization': 7.088607594936708, 'mem_free': 3342520320, 'swap_mem_total': 0, 'mem_total': 4390723584, 'services': [u'kv']} [2019-12-04 11:18:19,209] - [basetestcase] [140432460064512] - INFO - 172.23.104.199:8091 => {'swap_mem_used': 262144, 'cpu_utilization': 0.7556675062972292, 'mem_free': 3522031616, 'swap_mem_total': 3758092288, 'mem_total': 4201844736, 'services': [u'eventing']} [2019-12-04 11:18:19,210] - [basetestcase] [140432460064512] - INFO - 172.23.104.109:8091 => {'swap_mem_used': 80363520, 'cpu_utilization': 0.5025125628140703, 'mem_free': 3613949952, 'swap_mem_total': 3758092288, 'mem_total': 4220854272, 'services': [u'eventing']} [2019-12-04 11:18:19,211] - [basetestcase] [140432460064512] - INFO - 172.23.104.201:8091 => {'swap_mem_used': 0, 'cpu_utilization': 1.01010101010101, 'mem_free': 3485421568, 'swap_mem_total': 3758092288, 'mem_total': 4201848832, 'services': [u'index', u'n1ql']} [2019-12-04 11:18:19,212] - [basetestcase] [140432460064512] - INFO - 172.23.104.105:8091 => {'swap_mem_used': 741376, 'cpu_utilization': 1.259445843828715, 'mem_free': 3595030528, 'swap_mem_total': 3758092288, 'mem_total': 4220854272, 'services': [u'kv']} [2019-12-04 11:18:19,213] - [basetestcase] [140432460064512] - INFO - 172.23.104.103:8091 => {'swap_mem_used': 0, 'cpu_utilization': 7.323232323232324, 'mem_free': 3574587392, 'swap_mem_total': 0, 'mem_total': 4220149760, 'services': [u'kv']} [2019-12-04 11:18:19,214] - [basetestcase] [140432460064512] - INFO - --- End of cluster statistics --- [2019-12-04 11:18:19,223] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.100 with username:root [2019-12-04 11:18:19,395] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.100 [2019-12-04 11:18:40,554] - [basetestcase] [140432460064512] - INFO - list of n1ql nodes in cluster: [ip:172.23.104.201 port:8091 ssh_username:root] [2019-12-04 11:18:40,610] - [basetestcase] [140432460064512] - INFO - list of eventing nodes in cluster: [ip:172.23.104.107 port:8091 ssh_username:root, ip:172.23.104.199 port:8091 ssh_username:root, ip:172.23.104.109 port:8091 ssh_username:root] [2019-12-04 11:18:40,632] - [rest_client] [140432460064512] - INFO - settings/indexes params : storageMode=plasma [2019-12-04 11:18:40,653] - [eventing_base] [140432460064512] - INFO - Setting the min possible memory quota so that adding mode nodes to the cluster wouldn't be a problem. [2019-12-04 11:18:40,654] - [rest_client] [140432460064512] - INFO - pools/default params : memoryQuota=330 [2019-12-04 11:18:40,680] - [rest_client] [140432460064512] - INFO - pools/default params : indexMemoryQuota=256 [2019-12-04 11:18:40,705] - [rest_client] [140432460064512] - INFO - pools/default params : eventingMemoryQuota=512 [2019-12-04 11:18:40,786] - [basetestcase] [140432460064512] - INFO - list of n1ql nodes in cluster: [ip:172.23.104.201 port:8091 ssh_username:root] [2019-12-04 11:18:40,790] - [rest_client] [140432460064512] - INFO - pools/default params : memoryQuota=700 [2019-12-04 11:18:40,812] - [eventing_rebalance] [140432460064512] - INFO - 100 [2019-12-04 11:18:41,328] - [rest_client] [140424073172736] - INFO - http://172.23.104.100:8091/pools/default/buckets with param: bucketType=membase&threadsNumber=3&authType=none&compressionMode=passive&replicaIndex=1&name=src_bucket&evictionPolicy=valueOnly&flushEnabled=1&replicaNumber=0&ramQuotaMB=100 [2019-12-04 11:18:41,350] - [rest_client] [140424073172736] - INFO - 0.02 seconds to create bucket src_bucket [2019-12-04 11:18:41,350] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : src_bucket in 172.23.104.100 to accept set ops [2019-12-04 11:18:42,342] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:18:42,960] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 src_bucket [2019-12-04 11:18:43,259] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:18:43,652] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:18:43,935] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 src_bucket [2019-12-04 11:18:44,541] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:18:44,741] - [task] [140424073172736] - WARNING - vbucket map not ready after try 0 [2019-12-04 11:18:44,742] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : src_bucket in 172.23.104.100 to accept set ops [2019-12-04 11:18:45,091] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:18:45,346] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 src_bucket [2019-12-04 11:18:45,924] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:18:46,266] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:18:46,521] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 src_bucket [2019-12-04 11:18:46,765] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:18:46,980] - [task] [140424073172736] - WARNING - vbucket map not ready after try 1 [2019-12-04 11:18:46,981] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : src_bucket in 172.23.104.100 to accept set ops [2019-12-04 11:18:47,724] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:18:48,005] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 src_bucket [2019-12-04 11:18:48,314] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:18:48,721] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:18:49,281] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 src_bucket [2019-12-04 11:18:49,519] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:18:49,697] - [task] [140424073172736] - WARNING - vbucket map not ready after try 2 [2019-12-04 11:18:49,698] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : src_bucket in 172.23.104.100 to accept set ops [2019-12-04 11:18:50,423] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:18:50,695] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 src_bucket [2019-12-04 11:18:50,985] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:18:51,318] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:18:51,594] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 src_bucket [2019-12-04 11:18:52,221] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:18:52,405] - [task] [140424073172736] - WARNING - vbucket map not ready after try 3 [2019-12-04 11:18:52,406] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : src_bucket in 172.23.104.100 to accept set ops [2019-12-04 11:18:52,797] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:18:53,077] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 src_bucket [2019-12-04 11:18:53,652] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:18:53,989] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:18:54,247] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 src_bucket [2019-12-04 11:18:54,501] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:18:55,002] - [task] [140424073172736] - WARNING - vbucket map not ready after try 4 [2019-12-04 11:18:55,002] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : src_bucket in 172.23.104.100 to accept set ops [2019-12-04 11:18:55,390] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:18:55,674] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 src_bucket [2019-12-04 11:18:55,913] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:18:56,625] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:18:56,901] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 src_bucket [2019-12-04 11:18:57,124] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:18:57,302] - [task] [140424073172736] - WARNING - vbucket map not ready after try 5 [2019-12-04 11:18:58,428] - [rest_client] [140424073172736] - INFO - http://172.23.104.100:8091/pools/default/buckets with param: bucketType=membase&threadsNumber=3&authType=none&compressionMode=passive&replicaIndex=1&name=dst_bucket&evictionPolicy=valueOnly&flushEnabled=1&replicaNumber=0&ramQuotaMB=100 [2019-12-04 11:18:58,477] - [rest_client] [140424073172736] - INFO - 0.05 seconds to create bucket dst_bucket [2019-12-04 11:18:58,478] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : dst_bucket in 172.23.104.100 to accept set ops [2019-12-04 11:18:59,270] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 dst_bucket [2019-12-04 11:18:59,571] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 dst_bucket [2019-12-04 11:18:59,787] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 dst_bucket [2019-12-04 11:19:00,471] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 dst_bucket [2019-12-04 11:19:00,763] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 dst_bucket [2019-12-04 11:19:00,983] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 dst_bucket [2019-12-04 11:19:01,164] - [task] [140424073172736] - WARNING - vbucket map not ready after try 0 [2019-12-04 11:19:01,165] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : dst_bucket in 172.23.104.100 to accept set ops [2019-12-04 11:19:01,891] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 dst_bucket [2019-12-04 11:19:02,131] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 dst_bucket [2019-12-04 11:19:02,423] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 dst_bucket [2019-12-04 11:19:02,793] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 dst_bucket [2019-12-04 11:19:03,422] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 dst_bucket [2019-12-04 11:19:03,653] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 dst_bucket [2019-12-04 11:19:03,889] - [task] [140424073172736] - WARNING - vbucket map not ready after try 1 [2019-12-04 11:19:03,890] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : dst_bucket in 172.23.104.100 to accept set ops [2019-12-04 11:19:04,295] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 dst_bucket [2019-12-04 11:19:04,909] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 dst_bucket [2019-12-04 11:19:05,179] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 dst_bucket [2019-12-04 11:19:05,537] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 dst_bucket [2019-12-04 11:19:05,883] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 dst_bucket [2019-12-04 11:19:06,154] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 dst_bucket [2019-12-04 11:19:06,686] - [task] [140424073172736] - WARNING - vbucket map not ready after try 2 [2019-12-04 11:19:06,687] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : dst_bucket in 172.23.104.100 to accept set ops [2019-12-04 11:19:07,066] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 dst_bucket [2019-12-04 11:19:07,363] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 dst_bucket [2019-12-04 11:19:07,605] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 dst_bucket [2019-12-04 11:19:08,279] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 dst_bucket [2019-12-04 11:19:08,574] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 dst_bucket [2019-12-04 11:19:08,807] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 dst_bucket [2019-12-04 11:19:09,005] - [task] [140424073172736] - WARNING - vbucket map not ready after try 3 [2019-12-04 11:19:09,005] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : dst_bucket in 172.23.104.100 to accept set ops [2019-12-04 11:19:09,813] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 dst_bucket [2019-12-04 11:19:10,082] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 dst_bucket [2019-12-04 11:19:10,324] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 dst_bucket [2019-12-04 11:19:10,705] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 dst_bucket [2019-12-04 11:19:11,322] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 dst_bucket [2019-12-04 11:19:11,584] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 dst_bucket [2019-12-04 11:19:11,752] - [task] [140424073172736] - WARNING - vbucket map not ready after try 4 [2019-12-04 11:19:11,755] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : dst_bucket in 172.23.104.100 to accept set ops [2019-12-04 11:19:12,125] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 dst_bucket [2019-12-04 11:19:12,775] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 dst_bucket [2019-12-04 11:19:13,081] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 dst_bucket [2019-12-04 11:19:13,500] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 dst_bucket [2019-12-04 11:19:13,822] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 dst_bucket [2019-12-04 11:19:14,436] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 dst_bucket [2019-12-04 11:19:14,645] - [task] [140424073172736] - WARNING - vbucket map not ready after try 5 [2019-12-04 11:19:15,790] - [rest_client] [140424073172736] - INFO - http://172.23.104.100:8091/pools/default/buckets with param: bucketType=membase&threadsNumber=3&authType=none&compressionMode=passive&replicaIndex=1&name=metadata&evictionPolicy=valueOnly&flushEnabled=1&replicaNumber=0&ramQuotaMB=400 [2019-12-04 11:19:15,814] - [rest_client] [140424073172736] - INFO - 0.02 seconds to create bucket metadata [2019-12-04 11:19:15,815] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : metadata in 172.23.104.100 to accept set ops [2019-12-04 11:19:16,850] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 metadata [2019-12-04 11:19:17,197] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 metadata [2019-12-04 11:19:17,440] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 metadata [2019-12-04 11:19:18,185] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 metadata [2019-12-04 11:19:18,462] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 metadata [2019-12-04 11:19:18,696] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 metadata [2019-12-04 11:19:18,882] - [task] [140424073172736] - WARNING - vbucket map not ready after try 0 [2019-12-04 11:19:18,883] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : metadata in 172.23.104.100 to accept set ops [2019-12-04 11:19:19,591] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 metadata [2019-12-04 11:19:19,883] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 metadata [2019-12-04 11:19:20,150] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 metadata [2019-12-04 11:19:20,912] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 metadata [2019-12-04 11:19:21,177] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 metadata [2019-12-04 11:19:21,419] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 metadata [2019-12-04 11:19:21,619] - [task] [140424073172736] - WARNING - vbucket map not ready after try 1 [2019-12-04 11:19:21,620] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : metadata in 172.23.104.100 to accept set ops [2019-12-04 11:19:22,042] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 metadata [2019-12-04 11:19:22,658] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 metadata [2019-12-04 11:19:22,889] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 metadata [2019-12-04 11:19:23,277] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 metadata [2019-12-04 11:19:23,564] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 metadata [2019-12-04 11:19:24,225] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 metadata [2019-12-04 11:19:24,438] - [task] [140424073172736] - WARNING - vbucket map not ready after try 2 [2019-12-04 11:19:24,438] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : metadata in 172.23.104.100 to accept set ops [2019-12-04 11:19:24,888] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 metadata [2019-12-04 11:19:25,149] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 metadata [2019-12-04 11:19:25,719] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 metadata [2019-12-04 11:19:26,055] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 metadata [2019-12-04 11:19:26,326] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 metadata [2019-12-04 11:19:26,578] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 metadata [2019-12-04 11:19:26,746] - [task] [140424073172736] - WARNING - vbucket map not ready after try 3 [2019-12-04 11:19:26,747] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : metadata in 172.23.104.100 to accept set ops [2019-12-04 11:19:27,603] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 metadata [2019-12-04 11:19:27,917] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 metadata [2019-12-04 11:19:28,221] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 metadata [2019-12-04 11:19:29,017] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 metadata [2019-12-04 11:19:29,272] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 metadata [2019-12-04 11:19:29,541] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 metadata [2019-12-04 11:19:29,736] - [task] [140424073172736] - WARNING - vbucket map not ready after try 4 [2019-12-04 11:19:29,737] - [bucket_helper] [140424073172736] - INFO - waiting for memcached bucket : metadata in 172.23.104.100 to accept set ops [2019-12-04 11:19:30,486] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 metadata [2019-12-04 11:19:30,741] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 metadata [2019-12-04 11:19:31,000] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 metadata [2019-12-04 11:19:31,459] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.100:11210 metadata [2019-12-04 11:19:31,765] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.103:11210 metadata [2019-12-04 11:19:32,370] - [data_helper] [140424073172736] - INFO - creating direct client 172.23.104.105:11210 metadata [2019-12-04 11:19:32,569] - [task] [140424073172736] - WARNING - vbucket map not ready after try 5 [2019-12-04 11:19:33,014] - [eventing_base] [140432460064512] - INFO - Deploying the following handler code : Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio with {'buckets': [{'access': 'rw', 'alias': 'dst_bucket', 'bucket_name': 'dst_bucket'}], 'source_bucket': 'src_bucket', 'curl': [{'auth_type': 'no-auth', 'username': None, 'hostname': 'http://qa.sc.couchbase.com/', 'allow_cookies': False, 'value': 'server', 'password': None}], 'metadata_bucket': 'metadata'} [2019-12-04 11:19:33,015] - [eventing_base] [140432460064512] - INFO - function OnUpdate(doc, meta) { var request = { path : 'job/test_suite_executor/api/json?tree=jobs[component]' } try { var response = curl("GET", server, request); log('response body received from server:', response.body); log('response headers received from server:', response.headers); log('response status received from server:', response.status); var res= new Uint8Array(response.body); if(response.status == 200){ dst_bucket[meta.id]=response.body; } else{ dst_bucket[meta.id] = response.status; } } catch (e) { log('error:', e); } } function OnDelete(meta) { var request = { path : 'job/test_suite_executor/api/json?tree=jobs[component]' }; try { var response = curl("GET", server, request); log('response body received from server:', response.body); log('response headers received from server:', response.headers); log('response status received from server:', response.status); var res= new Uint8Array(response.body); if(response.status == 200){ delete dst_bucket[meta.id]; } else{ delete dst_bucket[meta.id]; } } catch (e) { log('error:', e); } } [2019-12-04 11:19:34,984] - [eventing_base] [140432460064512] - INFO - deploy Application : { "code": 0, "info": { "status": "Stored function: 'Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio' in metakv", "warnings": null } } [2019-12-04 11:19:34,985] - [basetestcase] [140432460064512] - INFO - sleep for 20 secs. Waiting for Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to deployed... ... [2019-12-04 11:19:55,937] - [basetestcase] [140432460064512] - INFO - sleep for 20 secs. Waiting for Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to deployed... ... [2019-12-04 11:20:18,656] - [basetestcase] [140432460064512] - INFO - create 40320 to src_bucket documents... [2019-12-04 11:20:18,920] - [data_helper] [140432460064512] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:20:19,601] - [data_helper] [140432460064512] - INFO - creating direct client 172.23.104.103:11210 src_bucket [2019-12-04 11:20:19,910] - [data_helper] [140432460064512] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:21:42,875] - [basetestcase] [140432460064512] - INFO - LOAD IS FINISHED [2019-12-04 11:21:44,094] - [rest_client] [140424073172736] - INFO - rebalance params : {'password': 'password', 'ejectedNodes': u'ns_1@172.23.104.103', 'user': 'Administrator', 'knownNodes': u'ns_1@172.23.104.107,ns_1@172.23.104.100,ns_1@172.23.104.199,ns_1@172.23.104.109,ns_1@172.23.104.201,ns_1@172.23.104.105,ns_1@172.23.104.103'} [2019-12-04 11:21:44,121] - [rest_client] [140424073172736] - INFO - rebalance operation started [2019-12-04 11:21:44,144] - [rest_client] [140424073172736] - INFO - rebalance percentage : 0.00 % [2019-12-04 11:21:44,145] - [task] [140424073172736] - INFO - Rebalance - status: running, progress: 0.00% [2019-12-04 11:21:45,953] - [rest_client] [140432460064512] - INFO - rebalance reached >100% in 3.02265119553 seconds [2019-12-04 11:21:54,241] - [rest_client] [140424073172736] - INFO - rebalance percentage : 12.52 % [2019-12-04 11:21:54,242] - [task] [140424073172736] - INFO - Rebalance - status: running, progress: 12.52% [2019-12-04 11:22:04,335] - [rest_client] [140424073172736] - INFO - rebalance percentage : 14.29 % [2019-12-04 11:22:04,335] - [task] [140424073172736] - INFO - Rebalance - status: running, progress: 14.29% [2019-12-04 11:22:14,420] - [rest_client] [140424073172736] - INFO - rebalance percentage : 24.08 % [2019-12-04 11:22:14,421] - [task] [140424073172736] - INFO - Rebalance - status: running, progress: 24.08% [2019-12-04 11:22:24,510] - [rest_client] [140424073172736] - INFO - rebalance percentage : 28.57 % [2019-12-04 11:22:24,511] - [task] [140424073172736] - INFO - Rebalance - status: running, progress: 28.57% [2019-12-04 11:22:34,580] - [rest_client] [140424073172736] - INFO - rebalance percentage : 34.89 % [2019-12-04 11:22:34,580] - [task] [140424073172736] - INFO - Rebalance - status: running, progress: 34.89% [2019-12-04 11:22:44,659] - [rest_client] [140424073172736] - INFO - rebalance percentage : 42.00 % [2019-12-04 11:22:44,660] - [task] [140424073172736] - INFO - Rebalance - status: running, progress: 42.00% [2019-12-04 11:22:54,727] - [rest_client] [140424073172736] - INFO - rebalance percentage : 50.00 % [2019-12-04 11:22:54,728] - [task] [140424073172736] - INFO - Rebalance - status: running, progress: 50.00% [2019-12-04 11:23:04,779] - [rest_client] [140424073172736] - INFO - rebalance percentage : 50.00 % [2019-12-04 11:23:04,780] - [task] [140424073172736] - INFO - Rebalance - status: running, progress: 50.00% [2019-12-04 11:23:14,839] - [task] [140424073172736] - INFO - Rebalance - status: none, progress: 100.00% [2019-12-04 11:23:14,915] - [task] [140424073172736] - INFO - rebalancing was completed with progress: 100% in 90.7932679653 sec [2019-12-04 11:23:14,965] - [basetestcase] [140432460064512] - INFO - list of eventing nodes in cluster: [ip:172.23.104.107 port:8091 ssh_username:root, ip:172.23.104.199 port:8091 ssh_username:root, ip:172.23.104.109 port:8091 ssh_username:root] [2019-12-04 11:23:15,035] - [basetestcase] [140432460064512] - INFO - list of eventing nodes in cluster: [ip:172.23.104.107 port:8091 ssh_username:root, ip:172.23.104.199 port:8091 ssh_username:root, ip:172.23.104.109 port:8091 ssh_username:root] [2019-12-04 11:23:15,253] - [eventing_base] [140432460064512] - INFO - Documents in source bucket : 40320 [2019-12-04 11:23:15,254] - [eventing_base] [140432460064512] - INFO - Final docs count... Current : 40320 Expected : 40320 [2019-12-04 11:23:17,546] - [eventing_base] [140432460064512] - INFO - Stats for Node 172.23.104.107 is [ { "dcp_feed_boundary": "everything", "event_processing_stats": { "adhoc_timer_response_received": 250, "agg_messages_sent_to_worker": 22175, "agg_queue_memory": 0, "agg_queue_memory_cap": 268435456, "agg_queue_size": 0, "agg_queue_size_cap": 800000, "agg_timer_feedback_queue_cap": 4000, "dcp_mutation": 13440, "dcp_mutation_sent_to_worker": 13440, "dcp_snapshot": 13440, "dcp_stream_req_counter": 342, "dcp_streamreq": 342, "execution_stats": 1669, "failure_stats": 1669, "latency_stats": 1669, "lcb_exception_stats": 1669, "log_level": 8, "num_processed_events": 13440, "processed_events_size": 9040812, "thr_count": 8, "thr_map": 8, "v8_init": 8, "v8_load": 8 }, "events_remaining": { "dcp_backlog": 0 }, "execution_stats": { "agg_queue_memory": 0, "agg_queue_size": 0, "curl": { "delete": 0, "get": 13440, "head": 0, "post": 0, "put": 0 }, "dcp_delete_msg_counter": 0, "dcp_delete_parse_failure": 0, "dcp_mutation_msg_counter": 13440, "dcp_mutation_parse_failure": 0, "enqueued_dcp_delete_msg_counter": 0, "enqueued_dcp_mutation_msg_counter": 13440, "enqueued_timer_msg_counter": 0, "feedback_queue_size": 0, "filtered_dcp_delete_counter": 0, "filtered_dcp_mutation_counter": 0, "lcb_retry_failure": 0, "messages_parsed": 22143, "num_processed_events": 13440, "on_delete_failure": 0, "on_delete_success": 0, "on_update_failure": 0, "on_update_success": 13440, "processed_events_size": 9040812, "timer_create_counter": 0, "timer_create_failure": 0, "timer_msg_counter": 0, "timer_responses_sent": 0, "timestamp": { "25305": "2019-12-04T19:23:15Z", "25313": "2019-12-04T19:23:14Z", "25314": "2019-12-04T19:23:14Z", "25327": "2019-12-04T19:23:14Z", "25341": "2019-12-04T19:23:14Z", "25343": "2019-12-04T19:23:14Z", "25345": "2019-12-04T19:23:14Z", "25362": "2019-12-04T19:23:14Z" }, "uv_msg_parse_failure": 0, "uv_try_write_failure_counter": 0 }, "failure_stats": { "app_worker_setting_events_lost": 0, "bucket_op_exception_count": 0, "checkpoint_failure_count": 0, "curl_non_200_response": 0, "dcp_events_lost": 0, "debugger_events_lost": 0, "delete_events_lost": 0, "mutation_events_lost": 0, "n1ql_op_exception_count": 0, "timeout_count": 0, "timer_callback_missing_counter": 0, "timer_context_size_exceeded_counter": 0, "timer_events_lost": 0, "timestamp": { "25305": "2019-12-04T19:23:15Z", "25313": "2019-12-04T19:23:14Z", "25314": "2019-12-04T19:23:14Z", "25327": "2019-12-04T19:23:14Z", "25341": "2019-12-04T19:23:14Z", "25343": "2019-12-04T19:23:14Z", "25345": "2019-12-04T19:23:14Z", "25362": "2019-12-04T19:23:14Z" }, "v8worker_events_lost": 0 }, "function_id": 2102568358, "function_name": "Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio", "gocb_creds_request_counter": 54, "internal_vb_distribution_stats": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[0-42]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[43-85]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[86-128]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[129-171]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[172-214]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[215-257]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[258-299]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[300-341]" }, "latency_percentile_stats": { "100": 205500, "50": 2700, "80": 3300, "90": 3900, "95": 4600, "99": 11100 }, "lcb_creds_request_counter": 384, "lcb_exception_stats": {}, "metastore_stats": {}, "planner_stats": [ { "host_name": "172.23.104.107:8096", "start_vb": 0, "vb_count": 342 }, { "host_name": "172.23.104.109:8096", "start_vb": 342, "vb_count": 341 }, { "host_name": "172.23.104.199:8096", "start_vb": 683, "vb_count": 341 } ], "vb_distribution_stats_from_metadata": { "172.23.104.107:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[0-42]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[43-85]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[86-128]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[129-171]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[172-214]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[215-257]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[258-299]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[300-341]" }, "172.23.104.109:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[342-384]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[385-427]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[428-470]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[471-513]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[514-556]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[557-598]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[599-640]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[641-682]" }, "172.23.104.199:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[683-725]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[726-768]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[769-811]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[812-854]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[855-897]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[898-939]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[940-981]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[982-1023]" } }, "worker_pids": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": 25305, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": 25313, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": 25314, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": 25327, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": 25343, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": 25345, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": 25341, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": 25362 } } ] [2019-12-04 11:23:21,112] - [eventing_base] [140432460064512] - INFO - Stats for Node 172.23.104.199 is [ { "dcp_feed_boundary": "everything", "event_processing_stats": { "adhoc_timer_response_received": 225, "agg_messages_sent_to_worker": 22282, "agg_queue_memory": 0, "agg_queue_memory_cap": 268435456, "agg_queue_size": 0, "agg_queue_size_cap": 800000, "agg_timer_feedback_queue_cap": 4000, "dcp_mutation": 13483, "dcp_mutation_sent_to_worker": 13483, "dcp_snapshot": 13483, "dcp_stream_req_counter": 341, "dcp_streamreq": 341, "execution_stats": 1682, "failure_stats": 1682, "latency_stats": 1682, "lcb_exception_stats": 1682, "log_level": 8, "num_processed_events": 13483, "processed_events_size": 9076888, "thr_count": 8, "thr_map": 8, "v8_init": 8, "v8_load": 8 }, "events_remaining": { "dcp_backlog": 0 }, "execution_stats": { "agg_queue_memory": 0, "agg_queue_size": 0, "curl": { "delete": 0, "get": 13483, "head": 0, "post": 0, "put": 0 }, "dcp_delete_msg_counter": 0, "dcp_delete_parse_failure": 0, "dcp_mutation_msg_counter": 13483, "dcp_mutation_parse_failure": 0, "enqueued_dcp_delete_msg_counter": 0, "enqueued_dcp_mutation_msg_counter": 13483, "enqueued_timer_msg_counter": 0, "feedback_queue_size": 0, "filtered_dcp_delete_counter": 0, "filtered_dcp_mutation_counter": 0, "lcb_retry_failure": 0, "messages_parsed": 22250, "num_processed_events": 13483, "on_delete_failure": 0, "on_delete_success": 0, "on_update_failure": 0, "on_update_success": 13483, "processed_events_size": 9076888, "timer_create_counter": 0, "timer_create_failure": 0, "timer_msg_counter": 0, "timer_responses_sent": 0, "timestamp": { "57402": "2019-12-04T19:23:17Z", "57407": "2019-12-04T19:23:17Z", "57412": "2019-12-04T19:23:17Z", "57433": "2019-12-04T19:23:17Z", "57438": "2019-12-04T19:23:17Z", "57455": "2019-12-04T19:23:18Z", "57456": "2019-12-04T19:23:17Z", "57462": "2019-12-04T19:23:17Z" }, "uv_msg_parse_failure": 0, "uv_try_write_failure_counter": 0 }, "failure_stats": { "app_worker_setting_events_lost": 0, "bucket_op_exception_count": 0, "checkpoint_failure_count": 0, "curl_non_200_response": 0, "dcp_events_lost": 0, "debugger_events_lost": 0, "delete_events_lost": 0, "mutation_events_lost": 0, "n1ql_op_exception_count": 0, "timeout_count": 0, "timer_callback_missing_counter": 0, "timer_context_size_exceeded_counter": 0, "timer_events_lost": 0, "timestamp": { "57402": "2019-12-04T19:23:17Z", "57407": "2019-12-04T19:23:17Z", "57412": "2019-12-04T19:23:17Z", "57433": "2019-12-04T19:23:17Z", "57438": "2019-12-04T19:23:17Z", "57455": "2019-12-04T19:23:18Z", "57456": "2019-12-04T19:23:17Z", "57462": "2019-12-04T19:23:17Z" }, "v8worker_events_lost": 0 }, "function_id": 2102568358, "function_name": "Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio", "gocb_creds_request_counter": 56, "internal_vb_distribution_stats": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[683-725]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[726-768]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[769-811]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[812-854]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[855-897]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[898-939]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[940-981]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[982-1023]" }, "latency_percentile_stats": { "100": 205200, "50": 2700, "80": 3300, "90": 3800, "95": 4400, "99": 11900 }, "lcb_creds_request_counter": 384, "lcb_exception_stats": {}, "metastore_stats": {}, "planner_stats": [ { "host_name": "172.23.104.107:8096", "start_vb": 0, "vb_count": 342 }, { "host_name": "172.23.104.109:8096", "start_vb": 342, "vb_count": 341 }, { "host_name": "172.23.104.199:8096", "start_vb": 683, "vb_count": 341 } ], "vb_distribution_stats_from_metadata": { "172.23.104.107:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[0-42]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[43-85]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[86-128]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[129-171]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[172-214]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[215-257]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[258-299]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[300-341]" }, "172.23.104.109:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[342-384]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[385-427]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[428-470]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[471-513]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[514-556]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[557-598]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[599-640]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[641-682]" }, "172.23.104.199:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[683-725]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[726-768]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[769-811]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[812-854]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[855-897]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[898-939]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[940-981]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[982-1023]" } }, "worker_pids": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": 57402, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": 57407, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": 57412, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": 57433, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": 57438, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": 57462, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": 57456, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": 57455 } } ] [2019-12-04 11:23:24,270] - [eventing_base] [140432460064512] - INFO - Stats for Node 172.23.104.109 is [ { "dcp_feed_boundary": "everything", "event_processing_stats": { "adhoc_timer_response_received": 230, "agg_messages_sent_to_worker": 23078, "agg_queue_memory": 0, "agg_queue_memory_cap": 268435456, "agg_queue_size": 0, "agg_queue_size_cap": 800000, "agg_timer_feedback_queue_cap": 4000, "dcp_mutation": 13397, "dcp_mutation_sent_to_worker": 13397, "dcp_snapshot": 13397, "dcp_stream_req_counter": 682, "dcp_streamend": 341, "dcp_streamreq": 682, "execution_stats": 1722, "failure_stats": 1722, "latency_stats": 1722, "lcb_exception_stats": 1722, "log_level": 8, "num_processed_events": 13397, "processed_events_size": 9017044, "thr_count": 8, "thr_map": 8, "v8_init": 8, "v8_load": 8 }, "events_remaining": { "dcp_backlog": 0 }, "execution_stats": { "agg_queue_memory": 0, "agg_queue_size": 0, "curl": { "delete": 0, "get": 13397, "head": 0, "post": 0, "put": 0 }, "dcp_delete_msg_counter": 0, "dcp_delete_parse_failure": 0, "dcp_mutation_msg_counter": 13397, "dcp_mutation_parse_failure": 0, "enqueued_dcp_delete_msg_counter": 0, "enqueued_dcp_mutation_msg_counter": 13397, "enqueued_timer_msg_counter": 0, "feedback_queue_size": 0, "filtered_dcp_delete_counter": 0, "filtered_dcp_mutation_counter": 0, "lcb_retry_failure": 0, "messages_parsed": 23046, "num_processed_events": 13397, "on_delete_failure": 0, "on_delete_success": 0, "on_update_failure": 0, "on_update_success": 13397, "processed_events_size": 9017044, "timer_create_counter": 0, "timer_create_failure": 0, "timer_msg_counter": 0, "timer_responses_sent": 0, "timestamp": { "5637": "2019-12-04T19:23:21Z", "5642": "2019-12-04T19:23:21Z", "5643": "2019-12-04T19:23:21Z", "5658": "2019-12-04T19:23:21Z", "5672": "2019-12-04T19:23:21Z", "5681": "2019-12-04T19:23:21Z", "5693": "2019-12-04T19:23:21Z", "5707": "2019-12-04T19:23:21Z" }, "uv_msg_parse_failure": 0, "uv_try_write_failure_counter": 0 }, "failure_stats": { "app_worker_setting_events_lost": 0, "bucket_op_exception_count": 0, "checkpoint_failure_count": 0, "curl_non_200_response": 0, "dcp_events_lost": 0, "debugger_events_lost": 0, "delete_events_lost": 0, "mutation_events_lost": 0, "n1ql_op_exception_count": 0, "timeout_count": 0, "timer_callback_missing_counter": 0, "timer_context_size_exceeded_counter": 0, "timer_events_lost": 0, "timestamp": { "5637": "2019-12-04T19:23:21Z", "5642": "2019-12-04T19:23:21Z", "5643": "2019-12-04T19:23:21Z", "5658": "2019-12-04T19:23:21Z", "5672": "2019-12-04T19:23:21Z", "5681": "2019-12-04T19:23:21Z", "5693": "2019-12-04T19:23:21Z", "5707": "2019-12-04T19:23:21Z" }, "v8worker_events_lost": 0 }, "function_id": 2102568358, "function_name": "Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio", "gocb_creds_request_counter": 54, "internal_vb_distribution_stats": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[342-384]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[385-427]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[428-470]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[471-513]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[514-556]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[557-598]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[599-640]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[641-682]" }, "latency_percentile_stats": { "100": 208200, "50": 2700, "80": 3200, "90": 3800, "95": 4400, "99": 10300 }, "lcb_creds_request_counter": 384, "lcb_exception_stats": {}, "metastore_stats": {}, "planner_stats": [ { "host_name": "172.23.104.107:8096", "start_vb": 0, "vb_count": 342 }, { "host_name": "172.23.104.109:8096", "start_vb": 342, "vb_count": 341 }, { "host_name": "172.23.104.199:8096", "start_vb": 683, "vb_count": 341 } ], "vb_distribution_stats_from_metadata": { "172.23.104.107:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[0-42]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[43-85]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[86-128]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[129-171]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[172-214]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[215-257]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[258-299]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[300-341]" }, "172.23.104.109:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[342-384]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[385-427]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[428-470]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[471-513]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[514-556]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[557-598]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[599-640]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[641-682]" }, "172.23.104.199:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[683-725]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[726-768]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[769-811]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[812-854]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[855-897]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[898-939]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[940-981]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[982-1023]" } }, "worker_pids": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": 5637, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": 5642, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": 5643, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": 5658, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": 5672, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": 5693, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": 5681, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": 5707 } } ] [2019-12-04 11:23:27,094] - [basetestcase] [140432460064512] - INFO - delete 40320 to src_bucket documents... [2019-12-04 11:23:27,295] - [data_helper] [140432460064512] - INFO - creating direct client 172.23.104.100:11210 src_bucket [2019-12-04 11:23:27,559] - [data_helper] [140432460064512] - INFO - creating direct client 172.23.104.105:11210 src_bucket [2019-12-04 11:24:25,528] - [basetestcase] [140432460064512] - INFO - LOAD IS FINISHED [2019-12-04 11:24:25,624] - [basetestcase] [140432460064512] - INFO - list of eventing nodes in cluster: [ip:172.23.104.107 port:8091 ssh_username:root, ip:172.23.104.199 port:8091 ssh_username:root, ip:172.23.104.109 port:8091 ssh_username:root] [2019-12-04 11:24:25,700] - [basetestcase] [140432460064512] - INFO - list of eventing nodes in cluster: [ip:172.23.104.107 port:8091 ssh_username:root, ip:172.23.104.199 port:8091 ssh_username:root, ip:172.23.104.109 port:8091 ssh_username:root] [2019-12-04 11:24:25,807] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 706 Expected : 0 ... [2019-12-04 11:24:55,938] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:25:26,064] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:25:56,213] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:26:26,364] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:26:56,486] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:27:26,626] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:27:56,773] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:28:26,880] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:28:57,022] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:29:27,151] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:29:57,287] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:30:27,429] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:30:57,568] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:31:27,699] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:31:57,836] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:32:27,969] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:32:58,097] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:33:28,240] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:33:58,374] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:34:28,506] - [basetestcase] [140432460064512] - INFO - sleep for 30 secs. Waiting for handler code Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio to complete bucket operations... Current : 26 Expected : 0 ... [2019-12-04 11:34:58,742] - [eventing_base] [140432460064512] - INFO - Documents in source bucket : 0 [2019-12-04 11:35:01,045] - [eventing_base] [140432460064512] - INFO - Stats for Node 172.23.104.107 is [ { "dcp_feed_boundary": "everything", "event_processing_stats": { "adhoc_timer_response_received": 411, "agg_messages_sent_to_worker": 63770, "agg_queue_memory": 0, "agg_queue_memory_cap": 268435456, "agg_queue_size": 0, "agg_queue_size_cap": 800000, "agg_timer_feedback_queue_cap": 4000, "dcp_deletion": 13440, "dcp_deletion_sent_to_worker": 13440, "dcp_mutation": 13440, "dcp_mutation_sent_to_worker": 13440, "dcp_snapshot": 26880, "dcp_stream_req_counter": 342, "dcp_streamreq": 342, "execution_stats": 7300, "failure_stats": 7300, "latency_stats": 7300, "lcb_exception_stats": 7300, "log_level": 8, "num_processed_events": 26880, "processed_events_size": 12605884, "thr_count": 8, "thr_map": 8, "v8_init": 8, "v8_load": 8 }, "events_remaining": { "dcp_backlog": 0 }, "execution_stats": { "agg_queue_memory": 0, "agg_queue_size": 0, "curl": { "delete": 0, "get": 26880, "head": 0, "post": 0, "put": 0 }, "dcp_delete_msg_counter": 13440, "dcp_delete_parse_failure": 0, "dcp_mutation_msg_counter": 13440, "dcp_mutation_parse_failure": 0, "enqueued_dcp_delete_msg_counter": 13440, "enqueued_dcp_mutation_msg_counter": 13440, "enqueued_timer_msg_counter": 0, "feedback_queue_size": 0, "filtered_dcp_delete_counter": 0, "filtered_dcp_mutation_counter": 0, "lcb_retry_failure": 0, "messages_parsed": 63738, "num_processed_events": 26880, "on_delete_failure": 0, "on_delete_success": 13440, "on_update_failure": 0, "on_update_success": 13440, "processed_events_size": 12605884, "timer_create_counter": 0, "timer_create_failure": 0, "timer_msg_counter": 0, "timer_responses_sent": 0, "timestamp": { "25305": "2019-12-04T19:34:58Z", "25313": "2019-12-04T19:34:58Z", "25314": "2019-12-04T19:34:58Z", "25327": "2019-12-04T19:34:58Z", "25341": "2019-12-04T19:34:58Z", "25343": "2019-12-04T19:34:58Z", "25345": "2019-12-04T19:34:58Z", "25362": "2019-12-04T19:34:58Z" }, "uv_msg_parse_failure": 0, "uv_try_write_failure_counter": 0 }, "failure_stats": { "app_worker_setting_events_lost": 0, "bucket_op_exception_count": 0, "checkpoint_failure_count": 0, "curl_non_200_response": 0, "dcp_events_lost": 0, "debugger_events_lost": 0, "delete_events_lost": 0, "mutation_events_lost": 0, "n1ql_op_exception_count": 0, "timeout_count": 0, "timer_callback_missing_counter": 0, "timer_context_size_exceeded_counter": 0, "timer_events_lost": 0, "timestamp": { "25305": "2019-12-04T19:34:58Z", "25313": "2019-12-04T19:34:58Z", "25314": "2019-12-04T19:34:58Z", "25327": "2019-12-04T19:34:58Z", "25341": "2019-12-04T19:34:58Z", "25343": "2019-12-04T19:34:58Z", "25345": "2019-12-04T19:34:58Z", "25362": "2019-12-04T19:34:58Z" }, "v8worker_events_lost": 0 }, "function_id": 2102568358, "function_name": "Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio", "gocb_creds_request_counter": 54, "internal_vb_distribution_stats": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[0-42]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[43-85]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[86-128]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[129-171]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[172-214]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[215-257]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[258-299]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[300-341]" }, "latency_percentile_stats": { "100": 857800, "50": 2700, "80": 3500, "90": 4500, "95": 5800, "99": 25600 }, "lcb_creds_request_counter": 384, "lcb_exception_stats": {}, "metastore_stats": {}, "planner_stats": [ { "host_name": "172.23.104.107:8096", "start_vb": 0, "vb_count": 342 }, { "host_name": "172.23.104.109:8096", "start_vb": 342, "vb_count": 341 }, { "host_name": "172.23.104.199:8096", "start_vb": 683, "vb_count": 341 } ], "vb_distribution_stats_from_metadata": { "172.23.104.107:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[0-42]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[43-85]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[86-128]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[129-171]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[172-214]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[215-257]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[258-299]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[300-341]" }, "172.23.104.109:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[342-384]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[385-427]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[428-470]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[471-513]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[514-556]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[557-598]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[599-640]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[641-682]" }, "172.23.104.199:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[683-725]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[726-768]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[769-811]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[812-854]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[855-897]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[898-939]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[940-981]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[982-1023]" } }, "worker_pids": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": 25305, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": 25313, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": 25314, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": 25327, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": 25343, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": 25345, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": 25341, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": 25362 } } ] [2019-12-04 11:35:06,245] - [eventing_base] [140432460064512] - INFO - Stats for Node 172.23.104.199 is [ { "dcp_feed_boundary": "everything", "event_processing_stats": { "adhoc_timer_response_received": 368, "agg_messages_sent_to_worker": 63915, "agg_queue_memory": 0, "agg_queue_memory_cap": 268435456, "agg_queue_size": 0, "agg_queue_size_cap": 800000, "agg_timer_feedback_queue_cap": 4000, "dcp_deletion": 13483, "dcp_deletion_sent_to_worker": 13483, "dcp_mutation": 13483, "dcp_mutation_sent_to_worker": 13483, "dcp_snapshot": 26966, "dcp_stream_req_counter": 341, "dcp_streamreq": 341, "execution_stats": 7312, "failure_stats": 7312, "latency_stats": 7312, "lcb_exception_stats": 7312, "log_level": 8, "num_processed_events": 26966, "processed_events_size": 12662148, "thr_count": 8, "thr_map": 8, "v8_init": 8, "v8_load": 8 }, "events_remaining": { "dcp_backlog": 0 }, "execution_stats": { "agg_queue_memory": 0, "agg_queue_size": 0, "curl": { "delete": 0, "get": 26966, "head": 0, "post": 0, "put": 0 }, "dcp_delete_msg_counter": 13483, "dcp_delete_parse_failure": 0, "dcp_mutation_msg_counter": 13483, "dcp_mutation_parse_failure": 0, "enqueued_dcp_delete_msg_counter": 13483, "enqueued_dcp_mutation_msg_counter": 13483, "enqueued_timer_msg_counter": 0, "feedback_queue_size": 0, "filtered_dcp_delete_counter": 0, "filtered_dcp_mutation_counter": 0, "lcb_retry_failure": 0, "messages_parsed": 63883, "num_processed_events": 26966, "on_delete_failure": 0, "on_delete_success": 13483, "on_update_failure": 0, "on_update_success": 13483, "processed_events_size": 12662148, "timer_create_counter": 0, "timer_create_failure": 0, "timer_msg_counter": 0, "timer_responses_sent": 0, "timestamp": { "57402": "2019-12-04T19:35:01Z", "57407": "2019-12-04T19:35:01Z", "57412": "2019-12-04T19:35:01Z", "57433": "2019-12-04T19:35:01Z", "57438": "2019-12-04T19:35:01Z", "57455": "2019-12-04T19:35:01Z", "57456": "2019-12-04T19:35:00Z", "57462": "2019-12-04T19:35:01Z" }, "uv_msg_parse_failure": 0, "uv_try_write_failure_counter": 0 }, "failure_stats": { "app_worker_setting_events_lost": 0, "bucket_op_exception_count": 0, "checkpoint_failure_count": 0, "curl_non_200_response": 0, "dcp_events_lost": 0, "debugger_events_lost": 0, "delete_events_lost": 0, "mutation_events_lost": 0, "n1ql_op_exception_count": 0, "timeout_count": 0, "timer_callback_missing_counter": 0, "timer_context_size_exceeded_counter": 0, "timer_events_lost": 0, "timestamp": { "57402": "2019-12-04T19:35:01Z", "57407": "2019-12-04T19:35:01Z", "57412": "2019-12-04T19:35:01Z", "57433": "2019-12-04T19:35:01Z", "57438": "2019-12-04T19:35:01Z", "57455": "2019-12-04T19:35:01Z", "57456": "2019-12-04T19:35:00Z", "57462": "2019-12-04T19:35:01Z" }, "v8worker_events_lost": 0 }, "function_id": 2102568358, "function_name": "Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio", "gocb_creds_request_counter": 56, "internal_vb_distribution_stats": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[683-725]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[726-768]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[769-811]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[812-854]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[855-897]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[898-939]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[940-981]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[982-1023]" }, "latency_percentile_stats": { "100": 707200, "50": 2700, "80": 3400, "90": 4200, "95": 5200, "99": 22600 }, "lcb_creds_request_counter": 384, "lcb_exception_stats": {}, "metastore_stats": {}, "planner_stats": [ { "host_name": "172.23.104.107:8096", "start_vb": 0, "vb_count": 342 }, { "host_name": "172.23.104.109:8096", "start_vb": 342, "vb_count": 341 }, { "host_name": "172.23.104.199:8096", "start_vb": 683, "vb_count": 341 } ], "vb_distribution_stats_from_metadata": { "172.23.104.107:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[0-42]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[43-85]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[86-128]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[129-171]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[172-214]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[215-257]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[258-299]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[300-341]" }, "172.23.104.109:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[342-384]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[385-427]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[428-470]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[471-513]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[514-556]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[557-598]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[599-640]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[641-682]" }, "172.23.104.199:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[683-725]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[726-768]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[769-811]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[812-854]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[855-897]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[898-939]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[940-981]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[982-1023]" } }, "worker_pids": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": 57402, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": 57407, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": 57412, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": 57433, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": 57438, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": 57462, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": 57456, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": 57455 } } ] [2019-12-04 11:35:09,342] - [eventing_base] [140432460064512] - INFO - Stats for Node 172.23.104.109 is [ { "dcp_feed_boundary": "everything", "event_processing_stats": { "adhoc_timer_response_received": 397, "agg_messages_sent_to_worker": 64675, "agg_queue_memory": 0, "agg_queue_memory_cap": 268435456, "agg_queue_size": 0, "agg_queue_size_cap": 800000, "agg_timer_feedback_queue_cap": 4000, "dcp_deletion": 13397, "dcp_deletion_sent_to_worker": 13397, "dcp_mutation": 13397, "dcp_mutation_sent_to_worker": 13397, "dcp_snapshot": 26794, "dcp_stream_req_counter": 682, "dcp_streamend": 341, "dcp_streamreq": 682, "execution_stats": 7362, "failure_stats": 7362, "latency_stats": 7362, "lcb_exception_stats": 7362, "log_level": 8, "num_processed_events": 26794, "processed_events_size": 12577796, "thr_count": 8, "thr_map": 8, "v8_init": 8, "v8_load": 8 }, "events_remaining": { "dcp_backlog": 0 }, "execution_stats": { "agg_queue_memory": 0, "agg_queue_size": 0, "curl": { "delete": 0, "get": 26794, "head": 0, "post": 0, "put": 0 }, "dcp_delete_msg_counter": 13397, "dcp_delete_parse_failure": 0, "dcp_mutation_msg_counter": 13397, "dcp_mutation_parse_failure": 0, "enqueued_dcp_delete_msg_counter": 13397, "enqueued_dcp_mutation_msg_counter": 13397, "enqueued_timer_msg_counter": 0, "feedback_queue_size": 0, "filtered_dcp_delete_counter": 0, "filtered_dcp_mutation_counter": 0, "lcb_retry_failure": 26, "messages_parsed": 64643, "num_processed_events": 26794, "on_delete_failure": 0, "on_delete_success": 13397, "on_update_failure": 0, "on_update_success": 13397, "processed_events_size": 12577796, "timer_create_counter": 0, "timer_create_failure": 0, "timer_msg_counter": 0, "timer_responses_sent": 0, "timestamp": { "5637": "2019-12-04T19:35:06Z", "5642": "2019-12-04T19:35:06Z", "5643": "2019-12-04T19:35:06Z", "5658": "2019-12-04T19:35:06Z", "5672": "2019-12-04T19:35:06Z", "5681": "2019-12-04T19:35:06Z", "5693": "2019-12-04T19:35:06Z", "5707": "2019-12-04T19:35:06Z" }, "uv_msg_parse_failure": 0, "uv_try_write_failure_counter": 0 }, "failure_stats": { "app_worker_setting_events_lost": 0, "bucket_op_exception_count": 26, "checkpoint_failure_count": 0, "curl_non_200_response": 0, "dcp_events_lost": 0, "debugger_events_lost": 0, "delete_events_lost": 0, "mutation_events_lost": 0, "n1ql_op_exception_count": 0, "timeout_count": 0, "timer_callback_missing_counter": 0, "timer_context_size_exceeded_counter": 0, "timer_events_lost": 0, "timestamp": { "5637": "2019-12-04T19:35:06Z", "5642": "2019-12-04T19:35:06Z", "5643": "2019-12-04T19:35:06Z", "5658": "2019-12-04T19:35:06Z", "5672": "2019-12-04T19:35:06Z", "5681": "2019-12-04T19:35:06Z", "5693": "2019-12-04T19:35:06Z", "5707": "2019-12-04T19:35:06Z" }, "v8worker_events_lost": 0 }, "function_id": 2102568358, "function_name": "Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio", "gocb_creds_request_counter": 54, "internal_vb_distribution_stats": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[342-384]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[385-427]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[428-470]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[471-513]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[514-556]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[557-598]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[599-640]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[641-682]" }, "latency_percentile_stats": { "100": 2698000, "50": 2700, "80": 3500, "90": 4400, "95": 5400, "99": 21300 }, "lcb_creds_request_counter": 2267, "lcb_exception_stats": { "2": 26 }, "metastore_stats": {}, "planner_stats": [ { "host_name": "172.23.104.107:8096", "start_vb": 0, "vb_count": 342 }, { "host_name": "172.23.104.109:8096", "start_vb": 342, "vb_count": 341 }, { "host_name": "172.23.104.199:8096", "start_vb": 683, "vb_count": 341 } ], "vb_distribution_stats_from_metadata": { "172.23.104.107:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[0-42]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[43-85]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[86-128]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[129-171]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[172-214]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[215-257]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[258-299]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[300-341]" }, "172.23.104.109:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[342-384]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[385-427]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[428-470]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[471-513]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[514-556]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[557-598]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[599-640]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[641-682]" }, "172.23.104.199:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[683-725]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[726-768]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[769-811]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[812-854]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[855-897]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[898-939]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[940-981]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[982-1023]" } }, "worker_pids": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": 5637, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": 5642, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": 5643, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": 5658, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": 5672, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": 5693, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": 5681, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": 5707 } } ] [2019-12-04 11:35:10,086] - [basetestcase] [140432460064512] - INFO - list of eventing nodes in cluster: [ip:172.23.104.107 port:8091 ssh_username:root, ip:172.23.104.199 port:8091 ssh_username:root, ip:172.23.104.109 port:8091 ssh_username:root] [2019-12-04 11:35:10,123] - [eventing_base] [140432460064512] - INFO - Go routine dumps for Node 172.23.104.107 is goroutine profile: total 400 24 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69ab 0xc8aeae 0x93cb11 # 0xc8aead github.com/couchbase/eventing/dcp.(*DcpFeed).genServer+0xcd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/dcp/upr.go:307 24 @ 0x90f27a 0x91ef90 0x10e50a0 0x93cb11 # 0x10e509f github.com/couchbase/eventing/consumer.(*Consumer).addToAggChan.func1+0x15f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:639 18 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa270b8 0xce24a0 0xce254a 0xcf6548 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa270b7 bufio.(*Reader).Read+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:216 # 0xce249f gopkg.in/couchbase/gocbcore%2ev7.(*memdTcpConn).readFullBuffer+0x8f /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdconn.go:143 # 0xce2549 gopkg.in/couchbase/gocbcore%2ev7.(*memdTcpConn).ReadPacket+0x49 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdconn.go:159 # 0xcf6547 gopkg.in/couchbase/gocbcore%2ev7.(*memdClient).run.func2+0xf7 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdclient.go:286 18 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69ab 0xcf7039 0x93cb11 # 0xcf7038 gopkg.in/couchbase/gocbcore%2ev7.(*memdPipelineClient).ioLoop.func1+0xf8 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdpipelineclient.go:63 18 @ 0x90f27a 0x90f32e 0x920d3b 0x942ff0 0xce3780 0xce3033 0xce4f5e 0xce5cf0 0x93cb11 # 0x920d3a sync.runtime_notifyListWait+0x10a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/sema.go:510 # 0x942fef sync.(*Cond).Wait+0x7f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/sync/cond.go:56 # 0xce377f gopkg.in/couchbase/gocbcore%2ev7.(*memdOpQueue).pop+0x3f /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdopqueue.go:138 # 0xce3032 gopkg.in/couchbase/gocbcore%2ev7.(*memdOpConsumer).Pop+0x32 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdopqueue.go:28 # 0xce4f5d gopkg.in/couchbase/gocbcore%2ev7.(*memdPipelineClient).ioLoop+0x1cd /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdpipelineclient.go:129 # 0xce5cef gopkg.in/couchbase/gocbcore%2ev7.(*memdPipelineClient).Run+0x31f /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdpipelineclient.go:195 18 @ 0x90f27a 0x91ef90 0xcf61c2 0x93cb11 # 0xcf61c1 gopkg.in/couchbase/gocbcore%2ev7.(*memdClient).run.func1+0xd1 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdclient.go:257 16 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0x965296 0x965408 0xc69679 0xc76c01 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0x965295 io.ReadAtLeast+0x85 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/io/io.go:309 # 0x965407 io.ReadFull+0x57 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/io/io.go:327 # 0xc69678 github.com/couchbase/eventing/dcp/transport.(*MCRequest).Receive+0x98 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/dcp/transport/mc_req.go:134 # 0xc76c00 github.com/couchbase/eventing/dcp/transport/client.(*DcpFeed).doReceive+0x220 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/dcp/transport/client/dcp_feed.go:1185 16 @ 0x90f27a 0x91ef90 0xc6de6b 0x93cb11 # 0xc6de6a github.com/couchbase/eventing/dcp/transport/client.(*DcpFeed).genServer+0x22a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/dcp/transport/client/dcp_feed.go:190 13 @ 0x90f27a 0x91ef90 0xb816ab 0x93cb11 # 0xb816aa net/http.(*persistConn).writeLoop+0x14a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1822 10 @ 0x90f27a 0x91ef90 0x1067343 0x93cb11 # 0x1067342 github.com/couchbase/eventing/suptree.(*Supervisor).Serve+0x2f2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/suptree/supervisor.go:293 9 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xb7f626 0xa26a2e 0xa2770c 0xb29894 0xb29372 0xb29653 0xb76c41 0xb76b8d 0xb82dec 0x9e08c2 0x9e05cd 0x9e0018 0xc45bd2 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xb7f625 net/http.(*persistConn).Read+0x135 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1453 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa2770b bufio.(*Reader).ReadSlice+0x2b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:341 # 0xb29893 net/http/internal.readChunkLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/internal/chunked.go:122 # 0xb29371 net/http/internal.(*chunkedReader).beginChunk+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/internal/chunked.go:48 # 0xb29652 net/http/internal.(*chunkedReader).Read+0x112 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/internal/chunked.go:93 # 0xb76c40 net/http.(*body).readLocked+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transfer.go:778 # 0xb76b8c net/http.(*body).Read+0xdc /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transfer.go:770 # 0xb82deb net/http.(*bodyEOFSignal).Read+0xdb /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:2187 # 0x9e08c1 encoding/json.(*Decoder).refill+0x131 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:159 # 0x9e05cc encoding/json.(*Decoder).readValue+0x23c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:134 # 0x9e0017 encoding/json.(*Decoder).Decode+0x77 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:63 # 0xc45bd1 github.com/couchbase/cbauth/metakv.doRunObserveChildren.func1+0x121 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:270 9 @ 0x90f27a 0x91ef90 0xb806a3 0x93cb11 # 0xb806a2 net/http.(*persistConn).readLoop+0x742 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1717 9 @ 0x90f27a 0x91ef90 0xcca98f 0x93cb11 # 0xcca98e gopkg.in/couchbase/gocbcore%2ev7.(*Agent).cccpLooper+0x55e /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/agentcccpcfg.go:17 9 @ 0x90f27a 0x91ef90 0xd302f4 0x93cb11 # 0xd302f3 gopkg.in/couchbase/gocb%2ev1.(*ThresholdLoggingTracer).loggerRoutine+0x133 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocb.v1/thresholdlogtracer.go:244 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0x9ae70e 0x9ab4aa 0xa26a2e 0xa2770c 0xa27924 0x10e023a 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0x9ae70d os.(*File).read+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/file_unix.go:226 # 0x9ab4a9 os.(*File).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/file.go:107 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa2770b bufio.(*Reader).ReadSlice+0x2b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:341 # 0xa27923 bufio.(*Reader).ReadLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:370 # 0x10e0239 github.com/couchbase/eventing/consumer.(*client).Serve.func1+0x1c9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/client.go:102 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0x9ae70e 0x9ab4aa 0xa26a2e 0xa2770c 0xa27924 0x10e04e2 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0x9ae70d os.(*File).read+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/file_unix.go:226 # 0x9ab4a9 os.(*File).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/file.go:107 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa2770b bufio.(*Reader).ReadSlice+0x2b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:341 # 0xa27923 bufio.(*Reader).ReadLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:370 # 0x10e04e1 github.com/couchbase/eventing/consumer.(*client).Serve.func2+0xd1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/client.go:115 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa26fac 0x1094875 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa26fab bufio.(*Reader).Read+0x12b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:202 # 0x1094874 github.com/couchbase/eventing/consumer.(*Consumer).feedbackReadMessageLoop+0x2b4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/handle_messages.go:728 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa26fac 0x1094f9a 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa26fab bufio.(*Reader).Read+0x12b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:202 # 0x1094f99 github.com/couchbase/eventing/consumer.(*Consumer).readMessageLoop+0x2a9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/handle_messages.go:792 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a7048 0xa6f172 0xa90012 0xa8dfd9 0x11142b1 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a7047 internal/poll.(*FD).Accept+0x1a7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:372 # 0xa6f171 net.(*netFD).accept+0x41 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:238 # 0xa90011 net.(*UnixListener).accept+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/unixsock_posix.go:162 # 0xa8dfd8 net.(*UnixListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/unixsock.go:253 # 0x11142b0 github.com/couchbase/eventing/producer.(*Producer).handleV8Consumer.func3.1+0x30 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:599 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a7048 0xa6f172 0xa90012 0xa8dfd9 0x1114c31 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a7047 internal/poll.(*FD).Accept+0x1a7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:372 # 0xa6f171 net.(*netFD).accept+0x41 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:238 # 0xa90011 net.(*UnixListener).accept+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/unixsock_posix.go:162 # 0xa8dfd8 net.(*UnixListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/unixsock.go:253 # 0x1114c30 github.com/couchbase/eventing/producer.(*Producer).handleV8Consumer.func4.1+0x30 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:633 8 @ 0x90f27a 0x90f32e 0x91fc14 0x91f839 0x945fa2 0x10bebaa 0x1069903 0x93cb11 # 0x91f838 sync.runtime_Semacquire+0x38 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/sema.go:56 # 0x945fa1 sync.(*WaitGroup).Wait+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/sync/waitgroup.go:129 # 0x10beba9 github.com/couchbase/eventing/consumer.(*Consumer).Serve+0x1c19 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/v8_consumer.go:292 # 0x1069902 github.com/couchbase/eventing/suptree.(*Supervisor).runService.func1+0x62 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/suptree/supervisor.go:413 8 @ 0x90f27a 0x91ef90 0x1077b8d 0x93cb11 # 0x1077b8c github.com/couchbase/eventing/consumer.(*Consumer).doLastSeqNoCheckpoint+0x19ac /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/checkpoint.go:24 8 @ 0x90f27a 0x91ef90 0x107bcec 0x93cb11 # 0x107bceb github.com/couchbase/eventing/consumer.(*Consumer).controlRoutine+0x1db /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/control_routine.go:21 8 @ 0x90f27a 0x91ef90 0x1092ade 0x93cb11 # 0x1092add github.com/couchbase/eventing/consumer.(*Consumer).sendMessageLoop+0x18d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/handle_messages.go:561 8 @ 0x90f27a 0x91ef90 0x1098377 0x93cb11 # 0x1098376 github.com/couchbase/eventing/consumer.(*Consumer).processDCPEvents+0x736 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:57 8 @ 0x90f27a 0x91ef90 0x109d9f8 0x93cb11 # 0x109d9f7 github.com/couchbase/eventing/consumer.(*Consumer).processStatsEvents+0x137 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:345 8 @ 0x90f27a 0x91ef90 0x109e883 0x93cb11 # 0x109e882 github.com/couchbase/eventing/consumer.(*Consumer).processFilterEvents+0x2e2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:395 8 @ 0x90f27a 0x91ef90 0x10a91fa 0x93cb11 # 0x10a91f9 github.com/couchbase/eventing/consumer.(*Consumer).handleFailoverLog+0x149 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:972 8 @ 0x90f27a 0x91ef90 0x10ad2be 0x93cb11 # 0x10ad2bd github.com/couchbase/eventing/consumer.(*Consumer).processReqStreamMessages+0x12d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:1197 8 @ 0x90f27a 0x91ef90 0x10b8f75 0x93cb11 # 0x10b8f74 github.com/couchbase/eventing/consumer.(*Consumer).updateWorkerStats+0x114 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/stats_updater.go:102 8 @ 0x90f27a 0x91ef90 0x1114485 0x93cb11 # 0x1114484 github.com/couchbase/eventing/producer.(*Producer).handleV8Consumer.func3+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:603 8 @ 0x90f27a 0x91ef90 0x1114e05 0x93cb11 # 0x1114e04 github.com/couchbase/eventing/producer.(*Producer).handleV8Consumer.func4+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:637 8 @ 0x950c45 0x9b1358 0x9aad3c 0x9aa2eb 0xfb298c 0x107a964 0x1069903 0x93cb11 # 0x950c44 syscall.Syscall6+0x4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/syscall/asm_linux_amd64.s:44 # 0x9b1357 os.(*Process).blockUntilWaitable+0x97 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/wait_waitid.go:31 # 0x9aad3b os.(*Process).wait+0x3b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/exec_unix.go:22 # 0x9aa2ea os.(*Process).Wait+0x2a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/exec.go:123 # 0xfb298b os/exec.(*Cmd).Wait+0x5b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/exec/exec.go:461 # 0x107a963 github.com/couchbase/eventing/consumer.(*client).Serve+0x1113 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/client.go:125 # 0x1069902 github.com/couchbase/eventing/suptree.(*Supervisor).runService.func1+0x62 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/suptree/supervisor.go:413 4 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xb7f626 0xa26a2e 0xa26b7a 0xb800e5 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xb7f625 net/http.(*persistConn).Read+0x135 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1453 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa26b79 bufio.(*Reader).Peek+0x39 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:132 # 0xb800e4 net/http.(*persistConn).readLoop+0x184 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1601 2 @ 0x90f27a 0x91ef90 0x104b591 0x93cb11 # 0x104b590 github.com/couchbase/eventing/util.(*Stats).updater+0x100 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/util/stats.go:54 2 @ 0x90f27a 0x91ef90 0x11168fc 0x93cb11 # 0x11168fb github.com/couchbase/cbauth/service.withTimeout.func1+0xdb /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:38 1 @ 0x8f1e12 0x923d26 0xd4d312 0x93cb11 # 0x923d25 os/signal.signal_recv+0xa5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/sigqueue.go:139 # 0xd4d311 os/signal.loop+0x21 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/signal/signal_unix.go:22 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa270b8 0xbfb0ed 0x9e08c2 0x9e05cd 0x9e0018 0xbf9c6f 0xbf7937 0xbf722a 0xbf6897 0xbfb659 0xbfbebc 0x111678d 0x114c36e 0x11687c0 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa270b7 bufio.(*Reader).Read+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:216 # 0xbfb0ec github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x4c /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:101 # 0x9e08c1 encoding/json.(*Decoder).refill+0x131 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:159 # 0x9e05cc encoding/json.(*Decoder).readValue+0x23c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:134 # 0x9e0017 encoding/json.(*Decoder).Decode+0x77 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:63 # 0xbf9c6e net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x7e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/jsonrpc/server.go:66 # 0xbf7936 net/rpc.(*Server).readRequestHeader+0x66 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:589 # 0xbf7229 net/rpc.(*Server).readRequest+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:549 # 0xbf6896 net/rpc.(*Server).ServeCodec+0x96 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:464 # 0xbfb658 github.com/couchbase/cbauth/revrpc.(*Service).Run+0x538 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:147 # 0xbfbebb github.com/couchbase/cbauth/revrpc.BabysitService+0x5b /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:243 # 0x111678c github.com/couchbase/cbauth/service.RegisterManager+0xbc /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:146 # 0x114c36d github.com/couchbase/eventing/service_manager.(*ServiceMgr).registerWithServer+0x11d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:445 # 0x11687bf github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func1+0xbf /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:84 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa270b8 0xbfb0ed 0x9e08c2 0x9e05cd 0x9e0018 0xbf9c6f 0xbf7937 0xbf722a 0xbf6897 0xbfb659 0xbfbebc 0xc3f340 0xc409f5 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa270b7 bufio.(*Reader).Read+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:216 # 0xbfb0ec github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x4c /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:101 # 0x9e08c1 encoding/json.(*Decoder).refill+0x131 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:159 # 0x9e05cc encoding/json.(*Decoder).readValue+0x23c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:134 # 0x9e0017 encoding/json.(*Decoder).Decode+0x77 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:63 # 0xbf9c6e net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x7e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/jsonrpc/server.go:66 # 0xbf7936 net/rpc.(*Server).readRequestHeader+0x66 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:589 # 0xbf7229 net/rpc.(*Server).readRequest+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:549 # 0xbf6896 net/rpc.(*Server).ServeCodec+0x96 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:464 # 0xbfb658 github.com/couchbase/cbauth/revrpc.(*Service).Run+0x538 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:147 # 0xbfbebb github.com/couchbase/cbauth/revrpc.BabysitService+0x5b /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:243 # 0xc3f33f github.com/couchbase/cbauth.runRPCForSvc+0xbf /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/default.go:57 # 0xc409f4 github.com/couchbase/cbauth.startDefault.func1+0x34 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/default.go:66 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xb6784a 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xb67849 net/http.(*connReader).backgroundRead+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:668 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xb67cf8 0xa26a2e 0xa26b7a 0xb6cf83 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xb67cf7 net/http.(*connReader).Read+0xf7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:764 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa26b79 bufio.(*Reader).Peek+0x39 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:132 # 0xb6cf82 net/http.(*conn).serve+0x7c2 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:1855 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a7048 0xa6f172 0xa894be 0xa87a99 0xb7213f 0xaecf77 0xb70ed5 0xb714eb 0xb71dde 0x1168f26 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a7047 internal/poll.(*FD).Accept+0x1a7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:372 # 0xa6f171 net.(*netFD).accept+0x41 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:238 # 0xa894bd net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/tcpsock_posix.go:136 # 0xa87a98 net.(*TCPListener).AcceptTCP+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/tcpsock.go:246 # 0xb7213e net/http.tcpKeepAliveListener.Accept+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:3216 # 0xaecf76 crypto/tls.(*listener).Accept+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/crypto/tls/tls.go:52 # 0xb70ed4 net/http.(*Server).Serve+0x1a4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2770 # 0xb714ea net/http.(*Server).ServeTLS+0x27a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2839 # 0xb71ddd net/http.(*Server).ListenAndServeTLS+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:3032 # 0x1168f25 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func4+0x3f5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:228 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a7048 0xa6f172 0xa894be 0xa87a99 0xb7213f 0xb70ed5 0xb70c29 0x11689a7 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a7047 internal/poll.(*FD).Accept+0x1a7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:372 # 0xa6f171 net.(*netFD).accept+0x41 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:238 # 0xa894bd net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/tcpsock_posix.go:136 # 0xa87a98 net.(*TCPListener).AcceptTCP+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/tcpsock.go:246 # 0xb7213e net/http.tcpKeepAliveListener.Accept+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:3216 # 0xb70ed4 net/http.(*Server).Serve+0x1a4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2770 # 0xb70c28 net/http.(*Server).ListenAndServe+0xa8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2711 # 0x11689a6 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func2+0x1b6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:186 1 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69ab 0x117ce25 0x118820c 0x90ee22 0x93cb11 # 0x117ce24 github.com/couchbase/eventing/supervisor.(*SuperSupervisor).HandleSupCmdMsg+0x74 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/supervisor/super_supervisor.go:652 # 0x118820b main.main+0x3fb goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:108 # 0x90ee21 runtime.main+0x211 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/proc.go:198 1 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69ab 0x1183093 0x93cb11 # 0x1183092 github.com/couchbase/eventing/supervisor.NewSuperSupervisor.func1+0x72 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/supervisor/super_supervisor.go:79 1 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69eb 0x10434c7 0x93cb11 # 0x10434c6 github.com/couchbase/eventing/util.(*vbSeqnosReader).Routine+0xf6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/util/dcp_seqno.go:100 1 @ 0x90f27a 0x90f32e 0x92b7d6 0x1044c23 0x93cb11 # 0x92b7d5 time.Sleep+0x165 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/time.go:102 # 0x1044c22 github.com/couchbase/eventing/util.pollForDeletedBuckets+0x82 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/util/dcp_seqno.go:325 1 @ 0x90f27a 0x90f32e 0x92b7d6 0x10fba14 0x93cb11 # 0x92b7d5 time.Sleep+0x165 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/time.go:102 # 0x10fba13 github.com/couchbase/eventing/producer.(*appLogCloser).cleanupTask+0x73 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/log_rotate.go:142 1 @ 0x90f27a 0x90f32e 0x92b7d6 0x11066dc 0x93cb11 # 0x92b7d5 time.Sleep+0x165 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/time.go:102 # 0x11066db github.com/couchbase/eventing/producer.(*Producer).updateStats+0x14b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:873 1 @ 0x90f27a 0x91ef90 0x10fe7e1 0x1069903 0x93cb11 # 0x10fe7e0 github.com/couchbase/eventing/producer.(*Producer).Serve+0x17a0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:198 # 0x1069902 github.com/couchbase/eventing/suptree.(*Supervisor).runService.func1+0x62 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/suptree/supervisor.go:413 1 @ 0x90f27a 0x91ef90 0x1106df5 0x93cb11 # 0x1106df4 github.com/couchbase/eventing/producer.(*Producer).pollForDeletedVbs+0x134 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:897 1 @ 0x90f27a 0x91ef90 0x114cf67 0x1151cf2 0x1116bc4 0x1115feb 0x11163e3 0x1117582 0x93a3cb 0x98c499 0x98ba14 0xbf5fbe 0x93cb11 # 0x114cf66 github.com/couchbase/eventing/service_manager.(*ServiceMgr).wait+0x2a6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:542 # 0x1151cf1 github.com/couchbase/eventing/service_manager.(*ServiceMgr).GetTaskList+0x1a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/service_manager.go:37 # 0x1116bc3 github.com/couchbase/cbauth/service.serviceAPI.GetTaskList.func1+0x73 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:99 # 0x1115fea github.com/couchbase/cbauth/service.withTimeout+0x7a /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:46 # 0x11163e2 github.com/couchbase/cbauth/service.serviceAPI.GetTaskList+0xd2 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:98 # 0x93a3ca runtime.call64+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/asm_amd64.s:574 # 0x98c498 reflect.Value.call+0x968 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/reflect/value.go:447 # 0x98ba13 reflect.Value.Call+0xa3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/reflect/value.go:308 # 0xbf5fbd net/rpc.(*service).call+0x14d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:384 1 @ 0x90f27a 0x91ef90 0x114cf67 0x1152482 0x1116a14 0x1115feb 0x1116283 0x11174b2 0x93a3cb 0x98c499 0x98ba14 0xbf5fbe 0x93cb11 # 0x114cf66 github.com/couchbase/eventing/service_manager.(*ServiceMgr).wait+0x2a6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:542 # 0x1152481 github.com/couchbase/eventing/service_manager.(*ServiceMgr).GetCurrentTopology+0x1a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/service_manager.go:90 # 0x1116a13 github.com/couchbase/cbauth/service.serviceAPI.GetCurrentTopology.func1+0x73 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:76 # 0x1115fea github.com/couchbase/cbauth/service.withTimeout+0x7a /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:46 # 0x1116282 github.com/couchbase/cbauth/service.serviceAPI.GetCurrentTopology+0xd2 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:75 # 0x93a3ca runtime.call64+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/asm_amd64.s:574 # 0x98c498 reflect.Value.call+0x968 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/reflect/value.go:447 # 0x98ba13 reflect.Value.Call+0xa3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/reflect/value.go:308 # 0xbf5fbd net/rpc.(*service).call+0x14d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:384 1 @ 0x90f27a 0x91ef90 0xb98e91 0x93cb11 # 0xb98e90 github.com/couchbase/cbauth/cbauthimpl.(*cfgChangeNotifier).loop+0xe0 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:272 1 @ 0x90f27a 0x91ef90 0xb992ff 0x93cb11 # 0xb992fe github.com/couchbase/cbauth/cbauthimpl.(*tlsNotifier).loop+0xce /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:352 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x116924a 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1169249 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func5+0x99 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:242 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x11693ca 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x11693c9 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func6+0x99 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:253 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x116954a 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1169549 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func7+0x99 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:264 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188939 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188938 main.main.func1+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:40 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188a49 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188a48 main.main.func2+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:52 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188b59 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188b58 main.main.func3+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:64 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188c69 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188c68 main.main.func4+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:76 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188d79 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188d78 main.main.func5+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:88 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188e89 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188e88 main.main.func6+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:100 1 @ 0xd44938 0xd44740 0xd41284 0xd4ca4d 0xd4cdd1 0x11463ad 0x1169fb8 0xb6ddf4 0xb6fa60 0xb70b0c 0xb6ce11 0x93cb11 # 0xd44937 runtime/pprof.writeRuntimeProfile+0x97 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/pprof/pprof.go:679 # 0xd4473f runtime/pprof.writeGoroutine+0x9f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/pprof/pprof.go:641 # 0xd41283 runtime/pprof.(*Profile).WriteTo+0x3e3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/pprof/pprof.go:310 # 0xd4ca4c net/http/pprof.handler.ServeHTTP+0x20c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/pprof/pprof.go:243 # 0xd4cdd0 net/http/pprof.Index+0x1d0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/pprof/pprof.go:254 # 0x11463ac github.com/couchbase/eventing/service_manager.(*ServiceMgr).indexHandler+0x8c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/http_handlers.go:3578 # 0x1169fb7 github.com/couchbase/eventing/service_manager.(*ServiceMgr).(github.com/couchbase/eventing/service_manager.indexHandler)-fm+0x47 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:99 # 0xb6ddf3 net/http.HandlerFunc.ServeHTTP+0x43 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:1947 # 0xb6fa5f net/http.(*ServeMux).ServeHTTP+0x12f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2337 # 0xb70b0b net/http.serverHandler.ServeHTTP+0xbb /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2694 # 0xb6ce10 net/http.(*conn).serve+0x650 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:1830 =================================================================================================================================================== [2019-12-04 11:35:10,170] - [eventing_base] [140432460064512] - INFO - Go routine dumps for Node 172.23.104.199 is goroutine profile: total 400 24 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69ab 0xc8aeae 0x93cb11 # 0xc8aead github.com/couchbase/eventing/dcp.(*DcpFeed).genServer+0xcd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/dcp/upr.go:307 24 @ 0x90f27a 0x91ef90 0x10e50a0 0x93cb11 # 0x10e509f github.com/couchbase/eventing/consumer.(*Consumer).addToAggChan.func1+0x15f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:639 18 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69ab 0xcf7039 0x93cb11 # 0xcf7038 gopkg.in/couchbase/gocbcore%2ev7.(*memdPipelineClient).ioLoop.func1+0xf8 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdpipelineclient.go:63 18 @ 0x90f27a 0x90f32e 0x920d3b 0x942ff0 0xce3780 0xce3033 0xce4f5e 0xce5cf0 0x93cb11 # 0x920d3a sync.runtime_notifyListWait+0x10a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/sema.go:510 # 0x942fef sync.(*Cond).Wait+0x7f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/sync/cond.go:56 # 0xce377f gopkg.in/couchbase/gocbcore%2ev7.(*memdOpQueue).pop+0x3f /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdopqueue.go:138 # 0xce3032 gopkg.in/couchbase/gocbcore%2ev7.(*memdOpConsumer).Pop+0x32 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdopqueue.go:28 # 0xce4f5d gopkg.in/couchbase/gocbcore%2ev7.(*memdPipelineClient).ioLoop+0x1cd /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdpipelineclient.go:129 # 0xce5cef gopkg.in/couchbase/gocbcore%2ev7.(*memdPipelineClient).Run+0x31f /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdpipelineclient.go:195 18 @ 0x90f27a 0x91ef90 0xcf61c2 0x93cb11 # 0xcf61c1 gopkg.in/couchbase/gocbcore%2ev7.(*memdClient).run.func1+0xd1 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdclient.go:257 17 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa270b8 0xce24a0 0xce254a 0xcf6548 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa270b7 bufio.(*Reader).Read+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:216 # 0xce249f gopkg.in/couchbase/gocbcore%2ev7.(*memdTcpConn).readFullBuffer+0x8f /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdconn.go:143 # 0xce2549 gopkg.in/couchbase/gocbcore%2ev7.(*memdTcpConn).ReadPacket+0x49 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdconn.go:159 # 0xcf6547 gopkg.in/couchbase/gocbcore%2ev7.(*memdClient).run.func2+0xf7 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdclient.go:286 16 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0x965296 0x965408 0xc69679 0xc76c01 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0x965295 io.ReadAtLeast+0x85 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/io/io.go:309 # 0x965407 io.ReadFull+0x57 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/io/io.go:327 # 0xc69678 github.com/couchbase/eventing/dcp/transport.(*MCRequest).Receive+0x98 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/dcp/transport/mc_req.go:134 # 0xc76c00 github.com/couchbase/eventing/dcp/transport/client.(*DcpFeed).doReceive+0x220 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/dcp/transport/client/dcp_feed.go:1185 16 @ 0x90f27a 0x91ef90 0xc6de6b 0x93cb11 # 0xc6de6a github.com/couchbase/eventing/dcp/transport/client.(*DcpFeed).genServer+0x22a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/dcp/transport/client/dcp_feed.go:190 13 @ 0x90f27a 0x91ef90 0xb816ab 0x93cb11 # 0xb816aa net/http.(*persistConn).writeLoop+0x14a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1822 10 @ 0x90f27a 0x91ef90 0x1067343 0x93cb11 # 0x1067342 github.com/couchbase/eventing/suptree.(*Supervisor).Serve+0x2f2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/suptree/supervisor.go:293 9 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xb7f626 0xa26a2e 0xa2770c 0xb29894 0xb29372 0xb29653 0xb76c41 0xb76b8d 0xb82dec 0x9e08c2 0x9e05cd 0x9e0018 0xc45bd2 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xb7f625 net/http.(*persistConn).Read+0x135 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1453 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa2770b bufio.(*Reader).ReadSlice+0x2b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:341 # 0xb29893 net/http/internal.readChunkLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/internal/chunked.go:122 # 0xb29371 net/http/internal.(*chunkedReader).beginChunk+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/internal/chunked.go:48 # 0xb29652 net/http/internal.(*chunkedReader).Read+0x112 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/internal/chunked.go:93 # 0xb76c40 net/http.(*body).readLocked+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transfer.go:778 # 0xb76b8c net/http.(*body).Read+0xdc /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transfer.go:770 # 0xb82deb net/http.(*bodyEOFSignal).Read+0xdb /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:2187 # 0x9e08c1 encoding/json.(*Decoder).refill+0x131 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:159 # 0x9e05cc encoding/json.(*Decoder).readValue+0x23c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:134 # 0x9e0017 encoding/json.(*Decoder).Decode+0x77 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:63 # 0xc45bd1 github.com/couchbase/cbauth/metakv.doRunObserveChildren.func1+0x121 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:270 9 @ 0x90f27a 0x91ef90 0xb806a3 0x93cb11 # 0xb806a2 net/http.(*persistConn).readLoop+0x742 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1717 9 @ 0x90f27a 0x91ef90 0xcca98f 0x93cb11 # 0xcca98e gopkg.in/couchbase/gocbcore%2ev7.(*Agent).cccpLooper+0x55e /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/agentcccpcfg.go:17 9 @ 0x90f27a 0x91ef90 0xd302f4 0x93cb11 # 0xd302f3 gopkg.in/couchbase/gocb%2ev1.(*ThresholdLoggingTracer).loggerRoutine+0x133 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocb.v1/thresholdlogtracer.go:244 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0x9ae70e 0x9ab4aa 0xa26a2e 0xa2770c 0xa27924 0x10e023a 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0x9ae70d os.(*File).read+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/file_unix.go:226 # 0x9ab4a9 os.(*File).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/file.go:107 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa2770b bufio.(*Reader).ReadSlice+0x2b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:341 # 0xa27923 bufio.(*Reader).ReadLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:370 # 0x10e0239 github.com/couchbase/eventing/consumer.(*client).Serve.func1+0x1c9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/client.go:102 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0x9ae70e 0x9ab4aa 0xa26a2e 0xa2770c 0xa27924 0x10e04e2 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0x9ae70d os.(*File).read+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/file_unix.go:226 # 0x9ab4a9 os.(*File).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/file.go:107 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa2770b bufio.(*Reader).ReadSlice+0x2b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:341 # 0xa27923 bufio.(*Reader).ReadLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:370 # 0x10e04e1 github.com/couchbase/eventing/consumer.(*client).Serve.func2+0xd1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/client.go:115 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa26fac 0x1094875 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa26fab bufio.(*Reader).Read+0x12b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:202 # 0x1094874 github.com/couchbase/eventing/consumer.(*Consumer).feedbackReadMessageLoop+0x2b4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/handle_messages.go:728 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa26fac 0x1094f9a 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa26fab bufio.(*Reader).Read+0x12b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:202 # 0x1094f99 github.com/couchbase/eventing/consumer.(*Consumer).readMessageLoop+0x2a9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/handle_messages.go:792 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a7048 0xa6f172 0xa90012 0xa8dfd9 0x11142b1 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a7047 internal/poll.(*FD).Accept+0x1a7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:372 # 0xa6f171 net.(*netFD).accept+0x41 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:238 # 0xa90011 net.(*UnixListener).accept+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/unixsock_posix.go:162 # 0xa8dfd8 net.(*UnixListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/unixsock.go:253 # 0x11142b0 github.com/couchbase/eventing/producer.(*Producer).handleV8Consumer.func3.1+0x30 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:599 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a7048 0xa6f172 0xa90012 0xa8dfd9 0x1114c31 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a7047 internal/poll.(*FD).Accept+0x1a7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:372 # 0xa6f171 net.(*netFD).accept+0x41 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:238 # 0xa90011 net.(*UnixListener).accept+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/unixsock_posix.go:162 # 0xa8dfd8 net.(*UnixListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/unixsock.go:253 # 0x1114c30 github.com/couchbase/eventing/producer.(*Producer).handleV8Consumer.func4.1+0x30 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:633 8 @ 0x90f27a 0x90f32e 0x91fc14 0x91f839 0x945fa2 0x10bebaa 0x1069903 0x93cb11 # 0x91f838 sync.runtime_Semacquire+0x38 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/sema.go:56 # 0x945fa1 sync.(*WaitGroup).Wait+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/sync/waitgroup.go:129 # 0x10beba9 github.com/couchbase/eventing/consumer.(*Consumer).Serve+0x1c19 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/v8_consumer.go:292 # 0x1069902 github.com/couchbase/eventing/suptree.(*Supervisor).runService.func1+0x62 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/suptree/supervisor.go:413 8 @ 0x90f27a 0x91ef90 0x107bcec 0x93cb11 # 0x107bceb github.com/couchbase/eventing/consumer.(*Consumer).controlRoutine+0x1db /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/control_routine.go:21 8 @ 0x90f27a 0x91ef90 0x1092ade 0x93cb11 # 0x1092add github.com/couchbase/eventing/consumer.(*Consumer).sendMessageLoop+0x18d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/handle_messages.go:561 8 @ 0x90f27a 0x91ef90 0x1098377 0x93cb11 # 0x1098376 github.com/couchbase/eventing/consumer.(*Consumer).processDCPEvents+0x736 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:57 8 @ 0x90f27a 0x91ef90 0x109d9f8 0x93cb11 # 0x109d9f7 github.com/couchbase/eventing/consumer.(*Consumer).processStatsEvents+0x137 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:345 8 @ 0x90f27a 0x91ef90 0x109e883 0x93cb11 # 0x109e882 github.com/couchbase/eventing/consumer.(*Consumer).processFilterEvents+0x2e2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:395 8 @ 0x90f27a 0x91ef90 0x10a91fa 0x93cb11 # 0x10a91f9 github.com/couchbase/eventing/consumer.(*Consumer).handleFailoverLog+0x149 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:972 8 @ 0x90f27a 0x91ef90 0x10ad2be 0x93cb11 # 0x10ad2bd github.com/couchbase/eventing/consumer.(*Consumer).processReqStreamMessages+0x12d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:1197 8 @ 0x90f27a 0x91ef90 0x10b8f75 0x93cb11 # 0x10b8f74 github.com/couchbase/eventing/consumer.(*Consumer).updateWorkerStats+0x114 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/stats_updater.go:102 8 @ 0x90f27a 0x91ef90 0x1114485 0x93cb11 # 0x1114484 github.com/couchbase/eventing/producer.(*Producer).handleV8Consumer.func3+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:603 8 @ 0x90f27a 0x91ef90 0x1114e05 0x93cb11 # 0x1114e04 github.com/couchbase/eventing/producer.(*Producer).handleV8Consumer.func4+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:637 8 @ 0x950c45 0x9b1358 0x9aad3c 0x9aa2eb 0xfb298c 0x107a964 0x1069903 0x93cb11 # 0x950c44 syscall.Syscall6+0x4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/syscall/asm_linux_amd64.s:44 # 0x9b1357 os.(*Process).blockUntilWaitable+0x97 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/wait_waitid.go:31 # 0x9aad3b os.(*Process).wait+0x3b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/exec_unix.go:22 # 0x9aa2ea os.(*Process).Wait+0x2a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/exec.go:123 # 0xfb298b os/exec.(*Cmd).Wait+0x5b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/exec/exec.go:461 # 0x107a963 github.com/couchbase/eventing/consumer.(*client).Serve+0x1113 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/client.go:125 # 0x1069902 github.com/couchbase/eventing/suptree.(*Supervisor).runService.func1+0x62 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/suptree/supervisor.go:413 7 @ 0x90f27a 0x91ef90 0x1077b8d 0x93cb11 # 0x1077b8c github.com/couchbase/eventing/consumer.(*Consumer).doLastSeqNoCheckpoint+0x19ac /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/checkpoint.go:24 4 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xb7f626 0xa26a2e 0xa26b7a 0xb800e5 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xb7f625 net/http.(*persistConn).Read+0x135 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1453 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa26b79 bufio.(*Reader).Peek+0x39 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:132 # 0xb800e4 net/http.(*persistConn).readLoop+0x184 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1601 2 @ 0x90f27a 0x91ef90 0x104b591 0x93cb11 # 0x104b590 github.com/couchbase/eventing/util.(*Stats).updater+0x100 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/util/stats.go:54 2 @ 0x90f27a 0x91ef90 0x11168fc 0x93cb11 # 0x11168fb github.com/couchbase/cbauth/service.withTimeout.func1+0xdb /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:38 1 @ 0x8f1e12 0x923d26 0xd4d312 0x93cb11 # 0x923d25 os/signal.signal_recv+0xa5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/sigqueue.go:139 # 0xd4d311 os/signal.loop+0x21 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/signal/signal_unix.go:22 1 @ 0x8f320f 0x8f3398 0xcf6673 0x93cb11 # 0xcf6672 gopkg.in/couchbase/gocbcore%2ev7.(*memdClient).run.func2+0x222 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdclient.go:349 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa270b8 0xbfb0ed 0x9e08c2 0x9e05cd 0x9e0018 0xbf9c6f 0xbf7937 0xbf722a 0xbf6897 0xbfb659 0xbfbebc 0x111678d 0x114c36e 0x11687c0 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa270b7 bufio.(*Reader).Read+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:216 # 0xbfb0ec github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x4c /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:101 # 0x9e08c1 encoding/json.(*Decoder).refill+0x131 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:159 # 0x9e05cc encoding/json.(*Decoder).readValue+0x23c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:134 # 0x9e0017 encoding/json.(*Decoder).Decode+0x77 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:63 # 0xbf9c6e net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x7e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/jsonrpc/server.go:66 # 0xbf7936 net/rpc.(*Server).readRequestHeader+0x66 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:589 # 0xbf7229 net/rpc.(*Server).readRequest+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:549 # 0xbf6896 net/rpc.(*Server).ServeCodec+0x96 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:464 # 0xbfb658 github.com/couchbase/cbauth/revrpc.(*Service).Run+0x538 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:147 # 0xbfbebb github.com/couchbase/cbauth/revrpc.BabysitService+0x5b /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:243 # 0x111678c github.com/couchbase/cbauth/service.RegisterManager+0xbc /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:146 # 0x114c36d github.com/couchbase/eventing/service_manager.(*ServiceMgr).registerWithServer+0x11d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:445 # 0x11687bf github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func1+0xbf /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:84 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa270b8 0xbfb0ed 0x9e08c2 0x9e05cd 0x9e0018 0xbf9c6f 0xbf7937 0xbf722a 0xbf6897 0xbfb659 0xbfbebc 0xc3f340 0xc409f5 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa270b7 bufio.(*Reader).Read+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:216 # 0xbfb0ec github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x4c /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:101 # 0x9e08c1 encoding/json.(*Decoder).refill+0x131 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:159 # 0x9e05cc encoding/json.(*Decoder).readValue+0x23c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:134 # 0x9e0017 encoding/json.(*Decoder).Decode+0x77 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:63 # 0xbf9c6e net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x7e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/jsonrpc/server.go:66 # 0xbf7936 net/rpc.(*Server).readRequestHeader+0x66 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:589 # 0xbf7229 net/rpc.(*Server).readRequest+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:549 # 0xbf6896 net/rpc.(*Server).ServeCodec+0x96 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:464 # 0xbfb658 github.com/couchbase/cbauth/revrpc.(*Service).Run+0x538 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:147 # 0xbfbebb github.com/couchbase/cbauth/revrpc.BabysitService+0x5b /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:243 # 0xc3f33f github.com/couchbase/cbauth.runRPCForSvc+0xbf /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/default.go:57 # 0xc409f4 github.com/couchbase/cbauth.startDefault.func1+0x34 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/default.go:66 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xb6784a 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xb67849 net/http.(*connReader).backgroundRead+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:668 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xb67cf8 0xa26a2e 0xa26b7a 0xb6cf83 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xb67cf7 net/http.(*connReader).Read+0xf7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:764 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa26b79 bufio.(*Reader).Peek+0x39 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:132 # 0xb6cf82 net/http.(*conn).serve+0x7c2 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:1855 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a7048 0xa6f172 0xa894be 0xa87a99 0xb7213f 0xaecf77 0xb70ed5 0xb714eb 0xb71dde 0x1168f26 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a7047 internal/poll.(*FD).Accept+0x1a7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:372 # 0xa6f171 net.(*netFD).accept+0x41 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:238 # 0xa894bd net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/tcpsock_posix.go:136 # 0xa87a98 net.(*TCPListener).AcceptTCP+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/tcpsock.go:246 # 0xb7213e net/http.tcpKeepAliveListener.Accept+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:3216 # 0xaecf76 crypto/tls.(*listener).Accept+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/crypto/tls/tls.go:52 # 0xb70ed4 net/http.(*Server).Serve+0x1a4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2770 # 0xb714ea net/http.(*Server).ServeTLS+0x27a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2839 # 0xb71ddd net/http.(*Server).ListenAndServeTLS+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:3032 # 0x1168f25 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func4+0x3f5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:228 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a7048 0xa6f172 0xa894be 0xa87a99 0xb7213f 0xb70ed5 0xb70c29 0x11689a7 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a7047 internal/poll.(*FD).Accept+0x1a7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:372 # 0xa6f171 net.(*netFD).accept+0x41 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:238 # 0xa894bd net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/tcpsock_posix.go:136 # 0xa87a98 net.(*TCPListener).AcceptTCP+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/tcpsock.go:246 # 0xb7213e net/http.tcpKeepAliveListener.Accept+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:3216 # 0xb70ed4 net/http.(*Server).Serve+0x1a4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2770 # 0xb70c28 net/http.(*Server).ListenAndServe+0xa8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2711 # 0x11689a6 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func2+0x1b6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:186 1 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69ab 0x117ce25 0x118820c 0x90ee22 0x93cb11 # 0x117ce24 github.com/couchbase/eventing/supervisor.(*SuperSupervisor).HandleSupCmdMsg+0x74 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/supervisor/super_supervisor.go:652 # 0x118820b main.main+0x3fb goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:108 # 0x90ee21 runtime.main+0x211 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/proc.go:198 1 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69ab 0x1183093 0x93cb11 # 0x1183092 github.com/couchbase/eventing/supervisor.NewSuperSupervisor.func1+0x72 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/supervisor/super_supervisor.go:79 1 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69eb 0x10434c7 0x93cb11 # 0x10434c6 github.com/couchbase/eventing/util.(*vbSeqnosReader).Routine+0xf6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/util/dcp_seqno.go:100 1 @ 0x90f27a 0x90f32e 0x92b7d6 0x1044c23 0x93cb11 # 0x92b7d5 time.Sleep+0x165 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/time.go:102 # 0x1044c22 github.com/couchbase/eventing/util.pollForDeletedBuckets+0x82 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/util/dcp_seqno.go:325 1 @ 0x90f27a 0x90f32e 0x92b7d6 0x10fba14 0x93cb11 # 0x92b7d5 time.Sleep+0x165 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/time.go:102 # 0x10fba13 github.com/couchbase/eventing/producer.(*appLogCloser).cleanupTask+0x73 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/log_rotate.go:142 1 @ 0x90f27a 0x90f32e 0x92b7d6 0x11066dc 0x93cb11 # 0x92b7d5 time.Sleep+0x165 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/time.go:102 # 0x11066db github.com/couchbase/eventing/producer.(*Producer).updateStats+0x14b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:873 1 @ 0x90f27a 0x91ef90 0x10fe7e1 0x1069903 0x93cb11 # 0x10fe7e0 github.com/couchbase/eventing/producer.(*Producer).Serve+0x17a0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:198 # 0x1069902 github.com/couchbase/eventing/suptree.(*Supervisor).runService.func1+0x62 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/suptree/supervisor.go:413 1 @ 0x90f27a 0x91ef90 0x1106df5 0x93cb11 # 0x1106df4 github.com/couchbase/eventing/producer.(*Producer).pollForDeletedVbs+0x134 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:897 1 @ 0x90f27a 0x91ef90 0x114cf67 0x1151cf2 0x1116bc4 0x1115feb 0x11163e3 0x1117582 0x93a3cb 0x98c499 0x98ba14 0xbf5fbe 0x93cb11 # 0x114cf66 github.com/couchbase/eventing/service_manager.(*ServiceMgr).wait+0x2a6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:542 # 0x1151cf1 github.com/couchbase/eventing/service_manager.(*ServiceMgr).GetTaskList+0x1a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/service_manager.go:37 # 0x1116bc3 github.com/couchbase/cbauth/service.serviceAPI.GetTaskList.func1+0x73 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:99 # 0x1115fea github.com/couchbase/cbauth/service.withTimeout+0x7a /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:46 # 0x11163e2 github.com/couchbase/cbauth/service.serviceAPI.GetTaskList+0xd2 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:98 # 0x93a3ca runtime.call64+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/asm_amd64.s:574 # 0x98c498 reflect.Value.call+0x968 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/reflect/value.go:447 # 0x98ba13 reflect.Value.Call+0xa3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/reflect/value.go:308 # 0xbf5fbd net/rpc.(*service).call+0x14d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:384 1 @ 0x90f27a 0x91ef90 0x114cf67 0x1152482 0x1116a14 0x1115feb 0x1116283 0x11174b2 0x93a3cb 0x98c499 0x98ba14 0xbf5fbe 0x93cb11 # 0x114cf66 github.com/couchbase/eventing/service_manager.(*ServiceMgr).wait+0x2a6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:542 # 0x1152481 github.com/couchbase/eventing/service_manager.(*ServiceMgr).GetCurrentTopology+0x1a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/service_manager.go:90 # 0x1116a13 github.com/couchbase/cbauth/service.serviceAPI.GetCurrentTopology.func1+0x73 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:76 # 0x1115fea github.com/couchbase/cbauth/service.withTimeout+0x7a /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:46 # 0x1116282 github.com/couchbase/cbauth/service.serviceAPI.GetCurrentTopology+0xd2 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:75 # 0x93a3ca runtime.call64+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/asm_amd64.s:574 # 0x98c498 reflect.Value.call+0x968 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/reflect/value.go:447 # 0x98ba13 reflect.Value.Call+0xa3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/reflect/value.go:308 # 0xbf5fbd net/rpc.(*service).call+0x14d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:384 1 @ 0x90f27a 0x91ef90 0xb98e91 0x93cb11 # 0xb98e90 github.com/couchbase/cbauth/cbauthimpl.(*cfgChangeNotifier).loop+0xe0 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:272 1 @ 0x90f27a 0x91ef90 0xb992ff 0x93cb11 # 0xb992fe github.com/couchbase/cbauth/cbauthimpl.(*tlsNotifier).loop+0xce /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:352 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x116924a 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1169249 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func5+0x99 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:242 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x11693ca 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x11693c9 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func6+0x99 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:253 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x116954a 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1169549 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func7+0x99 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:264 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188939 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188938 main.main.func1+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:40 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188a49 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188a48 main.main.func2+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:52 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188b59 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188b58 main.main.func3+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:64 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188c69 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188c68 main.main.func4+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:76 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188d79 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188d78 main.main.func5+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:88 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188e89 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188e88 main.main.func6+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:100 1 @ 0x90f27a 0x91ef90 0xd037b3 0xd03f5f 0xd02426 0x10d0499 0x104947a 0x10768ef 0x93cb11 # 0xd037b2 gopkg.in/couchbase/gocb%2ev1.(*opManager).Wait+0x112 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocb.v1/bucket_crud.go:163 # 0xd03f5e gopkg.in/couchbase/gocb%2ev1.(*Bucket).get+0x23e /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocb.v1/bucket_crud.go:210 # 0xd02425 gopkg.in/couchbase/gocb%2ev1.(*Bucket).Get+0xe5 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocb.v1/bucket_crud.go:13 # 0x10d0498 github.com/couchbase/eventing/consumer.glob..func5+0x428 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/bucket_ops.go:193 # 0x1049479 github.com/couchbase/eventing/util.Retry+0x139 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/util/retry.go:65 # 0x10768ee github.com/couchbase/eventing/consumer.(*Consumer).doLastSeqNoCheckpoint+0x70e /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/checkpoint.go:58 1 @ 0xd44938 0xd44740 0xd41284 0xd4ca4d 0xd4cdd1 0x11463ad 0x1169fb8 0xb6ddf4 0xb6fa60 0xb70b0c 0xb6ce11 0x93cb11 # 0xd44937 runtime/pprof.writeRuntimeProfile+0x97 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/pprof/pprof.go:679 # 0xd4473f runtime/pprof.writeGoroutine+0x9f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/pprof/pprof.go:641 # 0xd41283 runtime/pprof.(*Profile).WriteTo+0x3e3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/pprof/pprof.go:310 # 0xd4ca4c net/http/pprof.handler.ServeHTTP+0x20c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/pprof/pprof.go:243 # 0xd4cdd0 net/http/pprof.Index+0x1d0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/pprof/pprof.go:254 # 0x11463ac github.com/couchbase/eventing/service_manager.(*ServiceMgr).indexHandler+0x8c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/http_handlers.go:3578 # 0x1169fb7 github.com/couchbase/eventing/service_manager.(*ServiceMgr).(github.com/couchbase/eventing/service_manager.indexHandler)-fm+0x47 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:99 # 0xb6ddf3 net/http.HandlerFunc.ServeHTTP+0x43 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:1947 # 0xb6fa5f net/http.(*ServeMux).ServeHTTP+0x12f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2337 # 0xb70b0b net/http.serverHandler.ServeHTTP+0xbb /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2694 # 0xb6ce10 net/http.(*conn).serve+0x650 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:1830 =================================================================================================================================================== [2019-12-04 11:35:10,207] - [eventing_base] [140432460064512] - INFO - Go routine dumps for Node 172.23.104.109 is goroutine profile: total 387 18 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa270b8 0xce24a0 0xce254a 0xcf6548 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa270b7 bufio.(*Reader).Read+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:216 # 0xce249f gopkg.in/couchbase/gocbcore%2ev7.(*memdTcpConn).readFullBuffer+0x8f /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdconn.go:143 # 0xce2549 gopkg.in/couchbase/gocbcore%2ev7.(*memdTcpConn).ReadPacket+0x49 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdconn.go:159 # 0xcf6547 gopkg.in/couchbase/gocbcore%2ev7.(*memdClient).run.func2+0xf7 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdclient.go:286 18 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69ab 0xcf7039 0x93cb11 # 0xcf7038 gopkg.in/couchbase/gocbcore%2ev7.(*memdPipelineClient).ioLoop.func1+0xf8 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdpipelineclient.go:63 18 @ 0x90f27a 0x90f32e 0x920d3b 0x942ff0 0xce3780 0xce3033 0xce4f5e 0xce5cf0 0x93cb11 # 0x920d3a sync.runtime_notifyListWait+0x10a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/sema.go:510 # 0x942fef sync.(*Cond).Wait+0x7f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/sync/cond.go:56 # 0xce377f gopkg.in/couchbase/gocbcore%2ev7.(*memdOpQueue).pop+0x3f /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdopqueue.go:138 # 0xce3032 gopkg.in/couchbase/gocbcore%2ev7.(*memdOpConsumer).Pop+0x32 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdopqueue.go:28 # 0xce4f5d gopkg.in/couchbase/gocbcore%2ev7.(*memdPipelineClient).ioLoop+0x1cd /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdpipelineclient.go:129 # 0xce5cef gopkg.in/couchbase/gocbcore%2ev7.(*memdPipelineClient).Run+0x31f /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdpipelineclient.go:195 18 @ 0x90f27a 0x91ef90 0xcf61c2 0x93cb11 # 0xcf61c1 gopkg.in/couchbase/gocbcore%2ev7.(*memdClient).run.func1+0xd1 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/memdclient.go:257 17 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69ab 0xc8aeae 0x93cb11 # 0xc8aead github.com/couchbase/eventing/dcp.(*DcpFeed).genServer+0xcd /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/dcp/upr.go:307 17 @ 0x90f27a 0x91ef90 0x10e50a0 0x93cb11 # 0x10e509f github.com/couchbase/eventing/consumer.(*Consumer).addToAggChan.func1+0x15f /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:639 16 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0x965296 0x965408 0xc69679 0xc76c01 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0x965295 io.ReadAtLeast+0x85 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/io/io.go:309 # 0x965407 io.ReadFull+0x57 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/io/io.go:327 # 0xc69678 github.com/couchbase/eventing/dcp/transport.(*MCRequest).Receive+0x98 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/dcp/transport/mc_req.go:134 # 0xc76c00 github.com/couchbase/eventing/dcp/transport/client.(*DcpFeed).doReceive+0x220 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/dcp/transport/client/dcp_feed.go:1185 16 @ 0x90f27a 0x91ef90 0xc6de6b 0x93cb11 # 0xc6de6a github.com/couchbase/eventing/dcp/transport/client.(*DcpFeed).genServer+0x22a /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/dcp/transport/client/dcp_feed.go:190 13 @ 0x90f27a 0x91ef90 0xb816ab 0x93cb11 # 0xb816aa net/http.(*persistConn).writeLoop+0x14a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1822 10 @ 0x90f27a 0x91ef90 0x1067343 0x93cb11 # 0x1067342 github.com/couchbase/eventing/suptree.(*Supervisor).Serve+0x2f2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/suptree/supervisor.go:293 9 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xb7f626 0xa26a2e 0xa2770c 0xb29894 0xb29372 0xb29653 0xb76c41 0xb76b8d 0xb82dec 0x9e08c2 0x9e05cd 0x9e0018 0xc45bd2 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xb7f625 net/http.(*persistConn).Read+0x135 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1453 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa2770b bufio.(*Reader).ReadSlice+0x2b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:341 # 0xb29893 net/http/internal.readChunkLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/internal/chunked.go:122 # 0xb29371 net/http/internal.(*chunkedReader).beginChunk+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/internal/chunked.go:48 # 0xb29652 net/http/internal.(*chunkedReader).Read+0x112 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/internal/chunked.go:93 # 0xb76c40 net/http.(*body).readLocked+0x60 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transfer.go:778 # 0xb76b8c net/http.(*body).Read+0xdc /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transfer.go:770 # 0xb82deb net/http.(*bodyEOFSignal).Read+0xdb /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:2187 # 0x9e08c1 encoding/json.(*Decoder).refill+0x131 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:159 # 0x9e05cc encoding/json.(*Decoder).readValue+0x23c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:134 # 0x9e0017 encoding/json.(*Decoder).Decode+0x77 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:63 # 0xc45bd1 github.com/couchbase/cbauth/metakv.doRunObserveChildren.func1+0x121 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:270 9 @ 0x90f27a 0x91ef90 0xb806a3 0x93cb11 # 0xb806a2 net/http.(*persistConn).readLoop+0x742 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1717 9 @ 0x90f27a 0x91ef90 0xcca98f 0x93cb11 # 0xcca98e gopkg.in/couchbase/gocbcore%2ev7.(*Agent).cccpLooper+0x55e /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocbcore.v7/agentcccpcfg.go:17 9 @ 0x90f27a 0x91ef90 0xd302f4 0x93cb11 # 0xd302f3 gopkg.in/couchbase/gocb%2ev1.(*ThresholdLoggingTracer).loggerRoutine+0x133 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/gopkg.in/couchbase/gocb.v1/thresholdlogtracer.go:244 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0x9ae70e 0x9ab4aa 0xa26a2e 0xa2770c 0xa27924 0x10e023a 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0x9ae70d os.(*File).read+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/file_unix.go:226 # 0x9ab4a9 os.(*File).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/file.go:107 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa2770b bufio.(*Reader).ReadSlice+0x2b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:341 # 0xa27923 bufio.(*Reader).ReadLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:370 # 0x10e0239 github.com/couchbase/eventing/consumer.(*client).Serve.func1+0x1c9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/client.go:102 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0x9ae70e 0x9ab4aa 0xa26a2e 0xa2770c 0xa27924 0x10e04e2 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0x9ae70d os.(*File).read+0x4d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/file_unix.go:226 # 0x9ab4a9 os.(*File).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/file.go:107 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa2770b bufio.(*Reader).ReadSlice+0x2b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:341 # 0xa27923 bufio.(*Reader).ReadLine+0x33 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:370 # 0x10e04e1 github.com/couchbase/eventing/consumer.(*client).Serve.func2+0xd1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/client.go:115 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa26fac 0x1094875 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa26fab bufio.(*Reader).Read+0x12b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:202 # 0x1094874 github.com/couchbase/eventing/consumer.(*Consumer).feedbackReadMessageLoop+0x2b4 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/handle_messages.go:728 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa26fac 0x1094f9a 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa26fab bufio.(*Reader).Read+0x12b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:202 # 0x1094f99 github.com/couchbase/eventing/consumer.(*Consumer).readMessageLoop+0x2a9 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/handle_messages.go:792 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a7048 0xa6f172 0xa90012 0xa8dfd9 0x11142b1 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a7047 internal/poll.(*FD).Accept+0x1a7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:372 # 0xa6f171 net.(*netFD).accept+0x41 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:238 # 0xa90011 net.(*UnixListener).accept+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/unixsock_posix.go:162 # 0xa8dfd8 net.(*UnixListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/unixsock.go:253 # 0x11142b0 github.com/couchbase/eventing/producer.(*Producer).handleV8Consumer.func3.1+0x30 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:599 8 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a7048 0xa6f172 0xa90012 0xa8dfd9 0x1114c31 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a7047 internal/poll.(*FD).Accept+0x1a7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:372 # 0xa6f171 net.(*netFD).accept+0x41 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:238 # 0xa90011 net.(*UnixListener).accept+0x31 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/unixsock_posix.go:162 # 0xa8dfd8 net.(*UnixListener).Accept+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/unixsock.go:253 # 0x1114c30 github.com/couchbase/eventing/producer.(*Producer).handleV8Consumer.func4.1+0x30 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:633 8 @ 0x90f27a 0x90f32e 0x91fc14 0x91f839 0x945fa2 0x10bebaa 0x1069903 0x93cb11 # 0x91f838 sync.runtime_Semacquire+0x38 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/sema.go:56 # 0x945fa1 sync.(*WaitGroup).Wait+0x71 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/sync/waitgroup.go:129 # 0x10beba9 github.com/couchbase/eventing/consumer.(*Consumer).Serve+0x1c19 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/v8_consumer.go:292 # 0x1069902 github.com/couchbase/eventing/suptree.(*Supervisor).runService.func1+0x62 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/suptree/supervisor.go:413 8 @ 0x90f27a 0x91ef90 0x1077b8d 0x93cb11 # 0x1077b8c github.com/couchbase/eventing/consumer.(*Consumer).doLastSeqNoCheckpoint+0x19ac /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/checkpoint.go:24 8 @ 0x90f27a 0x91ef90 0x107bcec 0x93cb11 # 0x107bceb github.com/couchbase/eventing/consumer.(*Consumer).controlRoutine+0x1db /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/control_routine.go:21 8 @ 0x90f27a 0x91ef90 0x1092ade 0x93cb11 # 0x1092add github.com/couchbase/eventing/consumer.(*Consumer).sendMessageLoop+0x18d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/handle_messages.go:561 8 @ 0x90f27a 0x91ef90 0x1098377 0x93cb11 # 0x1098376 github.com/couchbase/eventing/consumer.(*Consumer).processDCPEvents+0x736 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:57 8 @ 0x90f27a 0x91ef90 0x109d9f8 0x93cb11 # 0x109d9f7 github.com/couchbase/eventing/consumer.(*Consumer).processStatsEvents+0x137 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:345 8 @ 0x90f27a 0x91ef90 0x109e883 0x93cb11 # 0x109e882 github.com/couchbase/eventing/consumer.(*Consumer).processFilterEvents+0x2e2 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:395 8 @ 0x90f27a 0x91ef90 0x10a91fa 0x93cb11 # 0x10a91f9 github.com/couchbase/eventing/consumer.(*Consumer).handleFailoverLog+0x149 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:972 8 @ 0x90f27a 0x91ef90 0x10ad2be 0x93cb11 # 0x10ad2bd github.com/couchbase/eventing/consumer.(*Consumer).processReqStreamMessages+0x12d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/process_events.go:1197 8 @ 0x90f27a 0x91ef90 0x10b8f75 0x93cb11 # 0x10b8f74 github.com/couchbase/eventing/consumer.(*Consumer).updateWorkerStats+0x114 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/stats_updater.go:102 8 @ 0x90f27a 0x91ef90 0x1114485 0x93cb11 # 0x1114484 github.com/couchbase/eventing/producer.(*Producer).handleV8Consumer.func3+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:603 8 @ 0x90f27a 0x91ef90 0x1114e05 0x93cb11 # 0x1114e04 github.com/couchbase/eventing/producer.(*Producer).handleV8Consumer.func4+0x164 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:637 8 @ 0x950c45 0x9b1358 0x9aad3c 0x9aa2eb 0xfb298c 0x107a964 0x1069903 0x93cb11 # 0x950c44 syscall.Syscall6+0x4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/syscall/asm_linux_amd64.s:44 # 0x9b1357 os.(*Process).blockUntilWaitable+0x97 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/wait_waitid.go:31 # 0x9aad3b os.(*Process).wait+0x3b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/exec_unix.go:22 # 0x9aa2ea os.(*Process).Wait+0x2a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/exec.go:123 # 0xfb298b os/exec.(*Cmd).Wait+0x5b /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/exec/exec.go:461 # 0x107a963 github.com/couchbase/eventing/consumer.(*client).Serve+0x1113 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/consumer/client.go:125 # 0x1069902 github.com/couchbase/eventing/suptree.(*Supervisor).runService.func1+0x62 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/suptree/supervisor.go:413 4 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xb7f626 0xa26a2e 0xa26b7a 0xb800e5 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xb7f625 net/http.(*persistConn).Read+0x135 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1453 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa26b79 bufio.(*Reader).Peek+0x39 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:132 # 0xb800e4 net/http.(*persistConn).readLoop+0x184 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/transport.go:1601 2 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xb67cf8 0xa26a2e 0xa26b7a 0xb6cf83 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xb67cf7 net/http.(*connReader).Read+0xf7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:764 # 0xa26a2d bufio.(*Reader).fill+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:100 # 0xa26b79 bufio.(*Reader).Peek+0x39 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:132 # 0xb6cf82 net/http.(*conn).serve+0x7c2 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:1855 2 @ 0x90f27a 0x91ef90 0x104b591 0x93cb11 # 0x104b590 github.com/couchbase/eventing/util.(*Stats).updater+0x100 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/util/stats.go:54 2 @ 0x90f27a 0x91ef90 0x11168fc 0x93cb11 # 0x11168fb github.com/couchbase/cbauth/service.withTimeout.func1+0xdb /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:38 1 @ 0x8f1e12 0x923d26 0xd4d312 0x93cb11 # 0x923d25 os/signal.signal_recv+0xa5 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/sigqueue.go:139 # 0xd4d311 os/signal.loop+0x21 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/os/signal/signal_unix.go:22 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa270b8 0xbfb0ed 0x9e08c2 0x9e05cd 0x9e0018 0xbf9c6f 0xbf7937 0xbf722a 0xbf6897 0xbfb659 0xbfbebc 0x111678d 0x114c36e 0x11687c0 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa270b7 bufio.(*Reader).Read+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:216 # 0xbfb0ec github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x4c /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:101 # 0x9e08c1 encoding/json.(*Decoder).refill+0x131 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:159 # 0x9e05cc encoding/json.(*Decoder).readValue+0x23c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:134 # 0x9e0017 encoding/json.(*Decoder).Decode+0x77 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:63 # 0xbf9c6e net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x7e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/jsonrpc/server.go:66 # 0xbf7936 net/rpc.(*Server).readRequestHeader+0x66 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:589 # 0xbf7229 net/rpc.(*Server).readRequest+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:549 # 0xbf6896 net/rpc.(*Server).ServeCodec+0x96 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:464 # 0xbfb658 github.com/couchbase/cbauth/revrpc.(*Service).Run+0x538 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:147 # 0xbfbebb github.com/couchbase/cbauth/revrpc.BabysitService+0x5b /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:243 # 0x111678c github.com/couchbase/cbauth/service.RegisterManager+0xbc /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:146 # 0x114c36d github.com/couchbase/eventing/service_manager.(*ServiceMgr).registerWithServer+0x11d /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:445 # 0x11687bf github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func1+0xbf /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:84 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xa270b8 0xbfb0ed 0x9e08c2 0x9e05cd 0x9e0018 0xbf9c6f 0xbf7937 0xbf722a 0xbf6897 0xbfb659 0xbfbebc 0xc3f340 0xc409f5 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xa270b7 bufio.(*Reader).Read+0x237 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/bufio/bufio.go:216 # 0xbfb0ec github.com/couchbase/cbauth/revrpc.(*minirwc).Read+0x4c /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:101 # 0x9e08c1 encoding/json.(*Decoder).refill+0x131 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:159 # 0x9e05cc encoding/json.(*Decoder).readValue+0x23c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:134 # 0x9e0017 encoding/json.(*Decoder).Decode+0x77 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/encoding/json/stream.go:63 # 0xbf9c6e net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader+0x7e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/jsonrpc/server.go:66 # 0xbf7936 net/rpc.(*Server).readRequestHeader+0x66 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:589 # 0xbf7229 net/rpc.(*Server).readRequest+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:549 # 0xbf6896 net/rpc.(*Server).ServeCodec+0x96 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:464 # 0xbfb658 github.com/couchbase/cbauth/revrpc.(*Service).Run+0x538 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:147 # 0xbfbebb github.com/couchbase/cbauth/revrpc.BabysitService+0x5b /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/revrpc/revrpc.go:243 # 0xc3f33f github.com/couchbase/cbauth.runRPCForSvc+0xbf /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/default.go:57 # 0xc409f4 github.com/couchbase/cbauth.startDefault.func1+0x34 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/default.go:66 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a5aad 0xa6e85f 0xa7fb3a 0xb6784a 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a5aac internal/poll.(*FD).Read+0x17c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:157 # 0xa6e85e net.(*netFD).Read+0x4e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:202 # 0xa7fb39 net.(*conn).Read+0x69 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/net.go:176 # 0xb67849 net/http.(*connReader).backgroundRead+0x59 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:668 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a7048 0xa6f172 0xa894be 0xa87a99 0xb7213f 0xaecf77 0xb70ed5 0xb714eb 0xb71dde 0x1168f26 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a7047 internal/poll.(*FD).Accept+0x1a7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:372 # 0xa6f171 net.(*netFD).accept+0x41 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:238 # 0xa894bd net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/tcpsock_posix.go:136 # 0xa87a98 net.(*TCPListener).AcceptTCP+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/tcpsock.go:246 # 0xb7213e net/http.tcpKeepAliveListener.Accept+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:3216 # 0xaecf76 crypto/tls.(*listener).Accept+0x36 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/crypto/tls/tls.go:52 # 0xb70ed4 net/http.(*Server).Serve+0x1a4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2770 # 0xb714ea net/http.(*Server).ServeTLS+0x27a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2839 # 0xb71ddd net/http.(*Server).ListenAndServeTLS+0x11d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:3032 # 0x1168f25 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func4+0x3f5 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:228 1 @ 0x90f27a 0x90a59a 0x909c17 0x9a4bcb 0x9a4c4d 0x9a7048 0xa6f172 0xa894be 0xa87a99 0xb7213f 0xb70ed5 0xb70c29 0x11689a7 0x93cb11 # 0x909c16 internal/poll.runtime_pollWait+0x56 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/netpoll.go:173 # 0x9a4bca internal/poll.(*pollDesc).wait+0x9a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:85 # 0x9a4c4c internal/poll.(*pollDesc).waitRead+0x3c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_poll_runtime.go:90 # 0x9a7047 internal/poll.(*FD).Accept+0x1a7 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/internal/poll/fd_unix.go:372 # 0xa6f171 net.(*netFD).accept+0x41 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/fd_unix.go:238 # 0xa894bd net.(*TCPListener).accept+0x2d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/tcpsock_posix.go:136 # 0xa87a98 net.(*TCPListener).AcceptTCP+0x48 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/tcpsock.go:246 # 0xb7213e net/http.tcpKeepAliveListener.Accept+0x2e /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:3216 # 0xb70ed4 net/http.(*Server).Serve+0x1a4 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2770 # 0xb70c28 net/http.(*Server).ListenAndServe+0xa8 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2711 # 0x11689a6 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func2+0x1b6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:186 1 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69ab 0x117ce25 0x118820c 0x90ee22 0x93cb11 # 0x117ce24 github.com/couchbase/eventing/supervisor.(*SuperSupervisor).HandleSupCmdMsg+0x74 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/supervisor/super_supervisor.go:652 # 0x118820b main.main+0x3fb goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:108 # 0x90ee21 runtime.main+0x211 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/proc.go:198 1 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69ab 0x1183093 0x93cb11 # 0x1183092 github.com/couchbase/eventing/supervisor.NewSuperSupervisor.func1+0x72 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/supervisor/super_supervisor.go:79 1 @ 0x90f27a 0x90f32e 0x8e6cf2 0x8e69eb 0x10434c7 0x93cb11 # 0x10434c6 github.com/couchbase/eventing/util.(*vbSeqnosReader).Routine+0xf6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/util/dcp_seqno.go:100 1 @ 0x90f27a 0x90f32e 0x92b7d6 0x1044c23 0x93cb11 # 0x92b7d5 time.Sleep+0x165 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/time.go:102 # 0x1044c22 github.com/couchbase/eventing/util.pollForDeletedBuckets+0x82 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/util/dcp_seqno.go:325 1 @ 0x90f27a 0x90f32e 0x92b7d6 0x10fba14 0x93cb11 # 0x92b7d5 time.Sleep+0x165 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/time.go:102 # 0x10fba13 github.com/couchbase/eventing/producer.(*appLogCloser).cleanupTask+0x73 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/log_rotate.go:142 1 @ 0x90f27a 0x90f32e 0x92b7d6 0x11066dc 0x93cb11 # 0x92b7d5 time.Sleep+0x165 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/time.go:102 # 0x11066db github.com/couchbase/eventing/producer.(*Producer).updateStats+0x14b /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:873 1 @ 0x90f27a 0x91ef90 0x10fe7e1 0x1069903 0x93cb11 # 0x10fe7e0 github.com/couchbase/eventing/producer.(*Producer).Serve+0x17a0 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:198 # 0x1069902 github.com/couchbase/eventing/suptree.(*Supervisor).runService.func1+0x62 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/suptree/supervisor.go:413 1 @ 0x90f27a 0x91ef90 0x1106df5 0x93cb11 # 0x1106df4 github.com/couchbase/eventing/producer.(*Producer).pollForDeletedVbs+0x134 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/producer/producer.go:897 1 @ 0x90f27a 0x91ef90 0x114cf67 0x1151cf2 0x1116bc4 0x1115feb 0x11163e3 0x1117582 0x93a3cb 0x98c499 0x98ba14 0xbf5fbe 0x93cb11 # 0x114cf66 github.com/couchbase/eventing/service_manager.(*ServiceMgr).wait+0x2a6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:542 # 0x1151cf1 github.com/couchbase/eventing/service_manager.(*ServiceMgr).GetTaskList+0x1a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/service_manager.go:37 # 0x1116bc3 github.com/couchbase/cbauth/service.serviceAPI.GetTaskList.func1+0x73 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:99 # 0x1115fea github.com/couchbase/cbauth/service.withTimeout+0x7a /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:46 # 0x11163e2 github.com/couchbase/cbauth/service.serviceAPI.GetTaskList+0xd2 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:98 # 0x93a3ca runtime.call64+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/asm_amd64.s:574 # 0x98c498 reflect.Value.call+0x968 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/reflect/value.go:447 # 0x98ba13 reflect.Value.Call+0xa3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/reflect/value.go:308 # 0xbf5fbd net/rpc.(*service).call+0x14d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:384 1 @ 0x90f27a 0x91ef90 0x114cf67 0x1152482 0x1116a14 0x1115feb 0x1116283 0x11174b2 0x93a3cb 0x98c499 0x98ba14 0xbf5fbe 0x93cb11 # 0x114cf66 github.com/couchbase/eventing/service_manager.(*ServiceMgr).wait+0x2a6 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:542 # 0x1152481 github.com/couchbase/eventing/service_manager.(*ServiceMgr).GetCurrentTopology+0x1a1 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/service_manager.go:90 # 0x1116a13 github.com/couchbase/cbauth/service.serviceAPI.GetCurrentTopology.func1+0x73 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:76 # 0x1115fea github.com/couchbase/cbauth/service.withTimeout+0x7a /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:46 # 0x1116282 github.com/couchbase/cbauth/service.serviceAPI.GetCurrentTopology+0xd2 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/service/revrpc.go:75 # 0x93a3ca runtime.call64+0x3a /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/asm_amd64.s:574 # 0x98c498 reflect.Value.call+0x968 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/reflect/value.go:447 # 0x98ba13 reflect.Value.Call+0xa3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/reflect/value.go:308 # 0xbf5fbd net/rpc.(*service).call+0x14d /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/rpc/server.go:384 1 @ 0x90f27a 0x91ef90 0xb98e91 0x93cb11 # 0xb98e90 github.com/couchbase/cbauth/cbauthimpl.(*cfgChangeNotifier).loop+0xe0 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:272 1 @ 0x90f27a 0x91ef90 0xb992ff 0x93cb11 # 0xb992fe github.com/couchbase/cbauth/cbauthimpl.(*tlsNotifier).loop+0xce /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/cbauthimpl/impl.go:352 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x116924a 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1169249 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func5+0x99 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:242 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x11693ca 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x11693c9 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func6+0x99 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:253 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x116954a 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1169549 github.com/couchbase/eventing/service_manager.(*ServiceMgr).initService.func7+0x99 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:264 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188939 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188938 main.main.func1+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:40 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188a49 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188a48 main.main.func2+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:52 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188b59 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188b58 main.main.func3+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:64 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188c69 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188c68 main.main.func4+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:76 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188d79 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188d78 main.main.func5+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:88 1 @ 0x90f27a 0x91ef90 0xc4333a 0xc42ff8 0xc43ab5 0x1188e89 0x93cb11 # 0xc43339 github.com/couchbase/cbauth/metakv.doRunObserveChildren+0x2f9 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:287 # 0xc42ff7 github.com/couchbase/cbauth/metakv.(*store).runObserveChildren+0x57 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:246 # 0xc43ab4 github.com/couchbase/cbauth/metakv.RunObserveChildren+0x54 /home/couchbase/jenkins/workspace/couchbase-server-unix/godeps/src/github.com/couchbase/cbauth/metakv/metakv.go:370 # 0x1188e88 main.main.func6+0x88 goproj/src/github.com/couchbase/eventing/cmd/producer/main.go:100 1 @ 0xd44938 0xd44740 0xd41284 0xd4ca4d 0xd4cdd1 0x11463ad 0x1169fb8 0xb6ddf4 0xb6fa60 0xb70b0c 0xb6ce11 0x93cb11 # 0xd44937 runtime/pprof.writeRuntimeProfile+0x97 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/pprof/pprof.go:679 # 0xd4473f runtime/pprof.writeGoroutine+0x9f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/pprof/pprof.go:641 # 0xd41283 runtime/pprof.(*Profile).WriteTo+0x3e3 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/runtime/pprof/pprof.go:310 # 0xd4ca4c net/http/pprof.handler.ServeHTTP+0x20c /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/pprof/pprof.go:243 # 0xd4cdd0 net/http/pprof.Index+0x1d0 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/pprof/pprof.go:254 # 0x11463ac github.com/couchbase/eventing/service_manager.(*ServiceMgr).indexHandler+0x8c /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/http_handlers.go:3578 # 0x1169fb7 github.com/couchbase/eventing/service_manager.(*ServiceMgr).(github.com/couchbase/eventing/service_manager.indexHandler)-fm+0x47 /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/service_manager/manager.go:99 # 0xb6ddf3 net/http.HandlerFunc.ServeHTTP+0x43 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:1947 # 0xb6fa5f net/http.(*ServeMux).ServeHTTP+0x12f /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2337 # 0xb70b0b net/http.serverHandler.ServeHTTP+0xbb /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:2694 # 0xb6ce10 net/http.(*conn).serve+0x650 /home/couchbase/.cbdepscache/exploded/x86_64/go-1.10.3/go/src/net/http/server.go:1830 =================================================================================================================================================== [2019-12-04 11:35:10,262] - [basetestcase] [140432460064512] - INFO - list of eventing nodes in cluster: [ip:172.23.104.107 port:8091 ssh_username:root, ip:172.23.104.199 port:8091 ssh_username:root, ip:172.23.104.109 port:8091 ssh_username:root] [2019-12-04 11:35:10,351] - [eventing_base] [140432460064512] - INFO - Stats for Node 172.23.104.107 is [ { "dcp_feed_boundary": "everything", "event_processing_stats": { "adhoc_timer_response_received": 411, "agg_messages_sent_to_worker": 64210, "agg_queue_memory": 0, "agg_queue_memory_cap": 268435456, "agg_queue_size": 0, "agg_queue_size_cap": 800000, "agg_timer_feedback_queue_cap": 4000, "dcp_deletion": 13440, "dcp_deletion_sent_to_worker": 13440, "dcp_mutation": 13440, "dcp_mutation_sent_to_worker": 13440, "dcp_snapshot": 26880, "dcp_stream_req_counter": 342, "dcp_streamreq": 342, "execution_stats": 7388, "failure_stats": 7388, "latency_stats": 7388, "lcb_exception_stats": 7388, "log_level": 8, "num_processed_events": 26880, "processed_events_size": 12605884, "thr_count": 8, "thr_map": 8, "v8_init": 8, "v8_load": 8 }, "events_remaining": { "dcp_backlog": 0 }, "execution_stats": { "agg_queue_memory": 0, "agg_queue_size": 0, "curl": { "delete": 0, "get": 26880, "head": 0, "post": 0, "put": 0 }, "dcp_delete_msg_counter": 13440, "dcp_delete_parse_failure": 0, "dcp_mutation_msg_counter": 13440, "dcp_mutation_parse_failure": 0, "enqueued_dcp_delete_msg_counter": 13440, "enqueued_dcp_mutation_msg_counter": 13440, "enqueued_timer_msg_counter": 0, "feedback_queue_size": 0, "filtered_dcp_delete_counter": 0, "filtered_dcp_mutation_counter": 0, "lcb_retry_failure": 0, "messages_parsed": 64183, "num_processed_events": 26880, "on_delete_failure": 0, "on_delete_success": 13440, "on_update_failure": 0, "on_update_success": 13440, "processed_events_size": 12605884, "timer_create_counter": 0, "timer_create_failure": 0, "timer_msg_counter": 0, "timer_responses_sent": 0, "timestamp": { "25305": "2019-12-04T19:35:10Z", "25313": "2019-12-04T19:35:09Z", "25314": "2019-12-04T19:35:09Z", "25327": "2019-12-04T19:35:09Z", "25341": "2019-12-04T19:35:09Z", "25343": "2019-12-04T19:35:09Z", "25345": "2019-12-04T19:35:09Z", "25362": "2019-12-04T19:35:09Z" }, "uv_msg_parse_failure": 0, "uv_try_write_failure_counter": 0 }, "failure_stats": { "app_worker_setting_events_lost": 0, "bucket_op_exception_count": 0, "checkpoint_failure_count": 0, "curl_non_200_response": 0, "dcp_events_lost": 0, "debugger_events_lost": 0, "delete_events_lost": 0, "mutation_events_lost": 0, "n1ql_op_exception_count": 0, "timeout_count": 0, "timer_callback_missing_counter": 0, "timer_context_size_exceeded_counter": 0, "timer_events_lost": 0, "timestamp": { "25305": "2019-12-04T19:35:10Z", "25313": "2019-12-04T19:35:09Z", "25314": "2019-12-04T19:35:09Z", "25327": "2019-12-04T19:35:09Z", "25341": "2019-12-04T19:35:09Z", "25343": "2019-12-04T19:35:09Z", "25345": "2019-12-04T19:35:09Z", "25362": "2019-12-04T19:35:09Z" }, "v8worker_events_lost": 0 }, "function_id": 2102568358, "function_name": "Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio", "gocb_creds_request_counter": 54, "internal_vb_distribution_stats": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[0-42]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[43-85]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[86-128]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[129-171]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[172-214]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[215-257]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[258-299]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[300-341]" }, "latency_percentile_stats": { "100": 857800, "50": 2700, "80": 3500, "90": 4500, "95": 5800, "99": 25600 }, "lcb_creds_request_counter": 384, "lcb_exception_stats": {}, "metastore_stats": {}, "planner_stats": [ { "host_name": "172.23.104.107:8096", "start_vb": 0, "vb_count": 342 }, { "host_name": "172.23.104.109:8096", "start_vb": 342, "vb_count": 341 }, { "host_name": "172.23.104.199:8096", "start_vb": 683, "vb_count": 341 } ], "vb_distribution_stats_from_metadata": { "172.23.104.107:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[0-42]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[43-85]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[86-128]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[129-171]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[172-214]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[215-257]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[258-299]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[300-341]" }, "172.23.104.109:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[342-384]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[385-427]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[428-470]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[471-513]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[514-556]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[557-598]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[599-640]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[641-682]" }, "172.23.104.199:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[683-725]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[726-768]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[769-811]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[812-854]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[855-897]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[898-939]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[940-981]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[982-1023]" } }, "worker_pids": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": 25305, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": 25313, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": 25314, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": 25327, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": 25343, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": 25345, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": 25341, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": 25362 } } ] [2019-12-04 11:35:10,419] - [eventing_base] [140432460064512] - INFO - Stats for Node 172.23.104.199 is [ { "dcp_feed_boundary": "everything", "event_processing_stats": { "adhoc_timer_response_received": 368, "agg_messages_sent_to_worker": 64245, "agg_queue_memory": 0, "agg_queue_memory_cap": 268435456, "agg_queue_size": 0, "agg_queue_size_cap": 800000, "agg_timer_feedback_queue_cap": 4000, "dcp_deletion": 13483, "dcp_deletion_sent_to_worker": 13483, "dcp_mutation": 13483, "dcp_mutation_sent_to_worker": 13483, "dcp_snapshot": 26966, "dcp_stream_req_counter": 341, "dcp_streamreq": 341, "execution_stats": 7378, "failure_stats": 7378, "latency_stats": 7378, "lcb_exception_stats": 7378, "log_level": 8, "num_processed_events": 26966, "processed_events_size": 12662148, "thr_count": 8, "thr_map": 8, "v8_init": 8, "v8_load": 8 }, "events_remaining": { "dcp_backlog": 0 }, "execution_stats": { "agg_queue_memory": 0, "agg_queue_size": 0, "curl": { "delete": 0, "get": 26966, "head": 0, "post": 0, "put": 0 }, "dcp_delete_msg_counter": 13483, "dcp_delete_parse_failure": 0, "dcp_mutation_msg_counter": 13483, "dcp_mutation_parse_failure": 0, "enqueued_dcp_delete_msg_counter": 13483, "enqueued_dcp_mutation_msg_counter": 13483, "enqueued_timer_msg_counter": 0, "feedback_queue_size": 0, "filtered_dcp_delete_counter": 0, "filtered_dcp_mutation_counter": 0, "lcb_retry_failure": 0, "messages_parsed": 64213, "num_processed_events": 26966, "on_delete_failure": 0, "on_delete_success": 13483, "on_update_failure": 0, "on_update_success": 13483, "processed_events_size": 12662148, "timer_create_counter": 0, "timer_create_failure": 0, "timer_msg_counter": 0, "timer_responses_sent": 0, "timestamp": { "57402": "2019-12-04T19:35:09Z", "57407": "2019-12-04T19:35:09Z", "57412": "2019-12-04T19:35:09Z", "57433": "2019-12-04T19:35:09Z", "57438": "2019-12-04T19:35:09Z", "57455": "2019-12-04T19:35:10Z", "57456": "2019-12-04T19:35:09Z", "57462": "2019-12-04T19:35:09Z" }, "uv_msg_parse_failure": 0, "uv_try_write_failure_counter": 0 }, "failure_stats": { "app_worker_setting_events_lost": 0, "bucket_op_exception_count": 0, "checkpoint_failure_count": 0, "curl_non_200_response": 0, "dcp_events_lost": 0, "debugger_events_lost": 0, "delete_events_lost": 0, "mutation_events_lost": 0, "n1ql_op_exception_count": 0, "timeout_count": 0, "timer_callback_missing_counter": 0, "timer_context_size_exceeded_counter": 0, "timer_events_lost": 0, "timestamp": { "57402": "2019-12-04T19:35:09Z", "57407": "2019-12-04T19:35:09Z", "57412": "2019-12-04T19:35:09Z", "57433": "2019-12-04T19:35:09Z", "57438": "2019-12-04T19:35:09Z", "57455": "2019-12-04T19:35:10Z", "57456": "2019-12-04T19:35:09Z", "57462": "2019-12-04T19:35:09Z" }, "v8worker_events_lost": 0 }, "function_id": 2102568358, "function_name": "Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio", "gocb_creds_request_counter": 56, "internal_vb_distribution_stats": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[683-725]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[726-768]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[769-811]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[812-854]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[855-897]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[898-939]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[940-981]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[982-1023]" }, "latency_percentile_stats": { "100": 707200, "50": 2700, "80": 3400, "90": 4200, "95": 5200, "99": 22600 }, "lcb_creds_request_counter": 384, "lcb_exception_stats": {}, "metastore_stats": {}, "planner_stats": [ { "host_name": "172.23.104.107:8096", "start_vb": 0, "vb_count": 342 }, { "host_name": "172.23.104.109:8096", "start_vb": 342, "vb_count": 341 }, { "host_name": "172.23.104.199:8096", "start_vb": 683, "vb_count": 341 } ], "vb_distribution_stats_from_metadata": { "172.23.104.107:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[0-42]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[43-85]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[86-128]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[129-171]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[172-214]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[215-257]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[258-299]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[300-341]" }, "172.23.104.109:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[342-384]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[385-427]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[428-470]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[471-513]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[514-556]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[557-598]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[599-640]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[641-682]" }, "172.23.104.199:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[683-725]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[726-768]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[769-811]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[812-854]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[855-897]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[898-939]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[940-981]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[982-1023]" } }, "worker_pids": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": 57402, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": 57407, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": 57412, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": 57433, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": 57438, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": 57462, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": 57456, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": 57455 } } ] [2019-12-04 11:35:10,513] - [eventing_base] [140432460064512] - INFO - Stats for Node 172.23.104.109 is [ { "dcp_feed_boundary": "everything", "event_processing_stats": { "adhoc_timer_response_received": 397, "agg_messages_sent_to_worker": 64810, "agg_queue_memory": 0, "agg_queue_memory_cap": 268435456, "agg_queue_size": 0, "agg_queue_size_cap": 800000, "agg_timer_feedback_queue_cap": 4000, "dcp_deletion": 13397, "dcp_deletion_sent_to_worker": 13397, "dcp_mutation": 13397, "dcp_mutation_sent_to_worker": 13397, "dcp_snapshot": 26794, "dcp_stream_req_counter": 682, "dcp_streamend": 341, "dcp_streamreq": 682, "execution_stats": 7389, "failure_stats": 7389, "latency_stats": 7389, "lcb_exception_stats": 7389, "log_level": 8, "num_processed_events": 26794, "processed_events_size": 12577796, "thr_count": 8, "thr_map": 8, "v8_init": 8, "v8_load": 8 }, "events_remaining": { "dcp_backlog": 0 }, "execution_stats": { "agg_queue_memory": 0, "agg_queue_size": 0, "curl": { "delete": 0, "get": 26794, "head": 0, "post": 0, "put": 0 }, "dcp_delete_msg_counter": 13397, "dcp_delete_parse_failure": 0, "dcp_mutation_msg_counter": 13397, "dcp_mutation_parse_failure": 0, "enqueued_dcp_delete_msg_counter": 13397, "enqueued_dcp_mutation_msg_counter": 13397, "enqueued_timer_msg_counter": 0, "feedback_queue_size": 0, "filtered_dcp_delete_counter": 0, "filtered_dcp_mutation_counter": 0, "lcb_retry_failure": 26, "messages_parsed": 64778, "num_processed_events": 26794, "on_delete_failure": 0, "on_delete_success": 13397, "on_update_failure": 0, "on_update_success": 13397, "processed_events_size": 12577796, "timer_create_counter": 0, "timer_create_failure": 0, "timer_msg_counter": 0, "timer_responses_sent": 0, "timestamp": { "5637": "2019-12-04T19:35:10Z", "5642": "2019-12-04T19:35:10Z", "5643": "2019-12-04T19:35:10Z", "5658": "2019-12-04T19:35:09Z", "5672": "2019-12-04T19:35:09Z", "5681": "2019-12-04T19:35:09Z", "5693": "2019-12-04T19:35:09Z", "5707": "2019-12-04T19:35:09Z" }, "uv_msg_parse_failure": 0, "uv_try_write_failure_counter": 0 }, "failure_stats": { "app_worker_setting_events_lost": 0, "bucket_op_exception_count": 26, "checkpoint_failure_count": 0, "curl_non_200_response": 0, "dcp_events_lost": 0, "debugger_events_lost": 0, "delete_events_lost": 0, "mutation_events_lost": 0, "n1ql_op_exception_count": 0, "timeout_count": 0, "timer_callback_missing_counter": 0, "timer_context_size_exceeded_counter": 0, "timer_events_lost": 0, "timestamp": { "5637": "2019-12-04T19:35:10Z", "5642": "2019-12-04T19:35:10Z", "5643": "2019-12-04T19:35:10Z", "5658": "2019-12-04T19:35:09Z", "5672": "2019-12-04T19:35:09Z", "5681": "2019-12-04T19:35:09Z", "5693": "2019-12-04T19:35:09Z", "5707": "2019-12-04T19:35:09Z" }, "v8worker_events_lost": 0 }, "function_id": 2102568358, "function_name": "Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio", "gocb_creds_request_counter": 54, "internal_vb_distribution_stats": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[342-384]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[385-427]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[428-470]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[471-513]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[514-556]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[557-598]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[599-640]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[641-682]" }, "latency_percentile_stats": { "100": 2698000, "50": 2700, "80": 3500, "90": 4400, "95": 5400, "99": 21300 }, "lcb_creds_request_counter": 2267, "lcb_exception_stats": { "2": 26 }, "metastore_stats": {}, "planner_stats": [ { "host_name": "172.23.104.107:8096", "start_vb": 0, "vb_count": 342 }, { "host_name": "172.23.104.109:8096", "start_vb": 342, "vb_count": 341 }, { "host_name": "172.23.104.199:8096", "start_vb": 683, "vb_count": 341 } ], "vb_distribution_stats_from_metadata": { "172.23.104.107:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[0-42]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[43-85]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[86-128]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[129-171]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[172-214]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[215-257]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[258-299]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[300-341]" }, "172.23.104.109:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[342-384]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[385-427]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[428-470]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[471-513]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[514-556]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[557-598]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[599-640]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[641-682]" }, "172.23.104.199:8096": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": "[683-725]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": "[726-768]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": "[769-811]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": "[812-854]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": "[855-897]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": "[898-939]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": "[940-981]", "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": "[982-1023]" } }, "worker_pids": { "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_0": 5637, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_1": 5642, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_2": 5643, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_3": 5658, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_4": 5672, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_5": 5693, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_6": 5681, "worker_Function_127387981_test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutatio_7": 5707 } } ] [2019-12-04 11:35:10,602] - [basetestcase] [140432460064512] - INFO - list of eventing nodes in cluster: [ip:172.23.104.107 port:8091 ssh_username:root, ip:172.23.104.199 port:8091 ssh_username:root, ip:172.23.104.109 port:8091 ssh_username:root] [2019-12-04 11:35:10,607] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.107 with username:root [2019-12-04 11:35:10,812] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.107 [2019-12-04 11:35:12,424] - [rest_client] [140432460064512] - INFO - /diag/eval status on 172.23.104.107:8091: True content: "/opt/couchbase/var/lib/couchbase/logs" command: filename:absname(element(2, application:get_env(ns_server,error_logger_mf_dir))). [2019-12-04 11:35:12,425] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.107: zgrep "panic" "/opt/couchbase/var/lib/couchbase/logs"/eventing.log* | wc -l [2019-12-04 11:35:13,067] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:13,068] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.107: ls "/opt/couchbase/var/lib/couchbase/logs"/../crash/| wc -l [2019-12-04 11:35:13,169] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:13,287] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.199 with username:root [2019-12-04 11:35:13,555] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.199 [2019-12-04 11:35:14,611] - [rest_client] [140432460064512] - INFO - /diag/eval status on 172.23.104.199:8091: True content: "/opt/couchbase/var/lib/couchbase/logs" command: filename:absname(element(2, application:get_env(ns_server,error_logger_mf_dir))). [2019-12-04 11:35:14,612] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.199: zgrep "panic" "/opt/couchbase/var/lib/couchbase/logs"/eventing.log* | wc -l [2019-12-04 11:35:14,831] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:14,831] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.199: ls "/opt/couchbase/var/lib/couchbase/logs"/../crash/| wc -l [2019-12-04 11:35:14,880] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:14,999] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.109 with username:root [2019-12-04 11:35:15,168] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.109 [2019-12-04 11:35:16,673] - [rest_client] [140432460064512] - INFO - /diag/eval status on 172.23.104.109:8091: True content: "/opt/couchbase/var/lib/couchbase/logs" command: filename:absname(element(2, application:get_env(ns_server,error_logger_mf_dir))). [2019-12-04 11:35:16,674] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.109: zgrep "panic" "/opt/couchbase/var/lib/couchbase/logs"/eventing.log* | wc -l [2019-12-04 11:35:17,249] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:17,250] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.109: ls "/opt/couchbase/var/lib/couchbase/logs"/../crash/| wc -l [2019-12-04 11:35:17,342] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:18,312] - [eventing_base] [140432460064512] - INFO - Bucket dst_bucket DGM is 100 [2019-12-04 11:35:18,416] - [eventing_base] [140432460064512] - INFO - Bucket metadata DGM is 100 [2019-12-04 11:35:18,513] - [eventing_base] [140432460064512] - INFO - Bucket src_bucket DGM is 100 [2019-12-04 11:35:18,563] - [basetestcase] [140432460064512] - INFO - ------- Cluster statistics ------- [2019-12-04 11:35:18,564] - [basetestcase] [140432460064512] - INFO - 172.23.104.107:8091 => {'swap_mem_used': 32768, 'cpu_utilization': 4.050632911392405, 'mem_free': 1919950848, 'swap_mem_total': 3758092288, 'mem_total': 4220854272, 'services': [u'eventing']} [2019-12-04 11:35:18,564] - [basetestcase] [140432460064512] - INFO - 172.23.104.100:8091 => {'swap_mem_used': 0, 'cpu_utilization': 11.31105398457584, 'mem_free': 3095007232, 'swap_mem_total': 0, 'mem_total': 4390723584, 'services': [u'kv']} [2019-12-04 11:35:18,564] - [basetestcase] [140432460064512] - INFO - 172.23.104.199:8091 => {'swap_mem_used': 524288, 'cpu_utilization': 4.102564102564102, 'mem_free': 1747697664, 'swap_mem_total': 3758092288, 'mem_total': 4201844736, 'services': [u'eventing']} [2019-12-04 11:35:18,565] - [basetestcase] [140432460064512] - INFO - 172.23.104.109:8091 => {'swap_mem_used': 80363520, 'cpu_utilization': 3.826530612244898, 'mem_free': 2005106688, 'swap_mem_total': 3758092288, 'mem_total': 4220854272, 'services': [u'eventing']} [2019-12-04 11:35:18,565] - [basetestcase] [140432460064512] - INFO - 172.23.104.201:8091 => {'swap_mem_used': 0, 'cpu_utilization': 1.005025125628141, 'mem_free': 3387654144, 'swap_mem_total': 3758092288, 'mem_total': 4201848832, 'services': [u'index', u'n1ql']} [2019-12-04 11:35:18,566] - [basetestcase] [140432460064512] - INFO - 172.23.104.105:8091 => {'swap_mem_used': 741376, 'cpu_utilization': 3.27455919395466, 'mem_free': 3361767424, 'swap_mem_total': 3758092288, 'mem_total': 4220854272, 'services': [u'kv']} [2019-12-04 11:35:18,566] - [basetestcase] [140432460064512] - INFO - --- End of cluster statistics --- [2019-12-04 11:35:18,570] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.100 with username:root [2019-12-04 11:35:18,773] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.100 [2019-12-04 11:35:19,826] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.103 with username:root [2019-12-04 11:35:19,961] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.103 [2019-12-04 11:35:21,084] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.105 with username:root [2019-12-04 11:35:21,258] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.105 [2019-12-04 11:35:22,493] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.107 with username:root [2019-12-04 11:35:22,665] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.107 [2019-12-04 11:35:24,422] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.109 with username:root [2019-12-04 11:35:24,591] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.109 [2019-12-04 11:35:26,334] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.199 with username:root [2019-12-04 11:35:26,503] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.199 [2019-12-04 11:35:27,880] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.201 with username:root [2019-12-04 11:35:28,047] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.201 [2019-12-04 11:35:29,214] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.202 with username:root [2019-12-04 11:35:29,383] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.202 [2019-12-04 11:35:30,700] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.100 with username:root [2019-12-04 11:35:30,871] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.100 [2019-12-04 11:35:31,717] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.100 with username:root [2019-12-04 11:35:31,889] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.100 [2019-12-04 11:35:32,756] - [remote_util] [140432460064512] - INFO - 172.23.104.100 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:35:33,003] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:35:33,004] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.100: systemctl stop couchbase-server.service [2019-12-04 11:35:35,835] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:35,837] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:35:35,902] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.100: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:35:36,525] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:36,527] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.100: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:35:36,581] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:36,585] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.100 with username:root [2019-12-04 11:35:36,721] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.100 [2019-12-04 11:35:37,555] - [remote_util] [140432460064512] - INFO - 172.23.104.100 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:35:37,767] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:35:37,768] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.100: systemctl start couchbase-server.service [2019-12-04 11:35:37,806] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:37,806] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:35:37,925] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.103 with username:root [2019-12-04 11:35:38,131] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.103 [2019-12-04 11:35:39,095] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.103 with username:root [2019-12-04 11:35:39,195] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.103 [2019-12-04 11:35:40,175] - [remote_util] [140432460064512] - INFO - 172.23.104.103 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:35:40,384] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:35:40,384] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.103: systemctl stop couchbase-server.service [2019-12-04 11:35:41,959] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:41,960] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:35:42,077] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.103: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:35:42,098] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:42,098] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.103: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:35:42,178] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:42,185] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.103 with username:root [2019-12-04 11:35:42,321] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.103 [2019-12-04 11:35:43,213] - [remote_util] [140432460064512] - INFO - 172.23.104.103 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:35:43,483] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:35:43,484] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.103: systemctl start couchbase-server.service [2019-12-04 11:35:43,521] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:43,522] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:35:43,647] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.105 with username:root [2019-12-04 11:35:43,784] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.105 [2019-12-04 11:35:44,730] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.105 with username:root [2019-12-04 11:35:44,868] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.105 [2019-12-04 11:35:45,892] - [remote_util] [140432460064512] - INFO - 172.23.104.105 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:35:46,176] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:35:46,178] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.105: systemctl stop couchbase-server.service [2019-12-04 11:35:48,163] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:48,165] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:35:48,230] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.105: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:35:48,501] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:48,501] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.105: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:35:48,553] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:48,562] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.105 with username:root [2019-12-04 11:35:48,700] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.105 [2019-12-04 11:35:49,746] - [remote_util] [140432460064512] - INFO - 172.23.104.105 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:35:50,003] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:35:50,004] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.105: systemctl start couchbase-server.service [2019-12-04 11:35:50,041] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:50,042] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:35:50,166] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.107 with username:root [2019-12-04 11:35:50,338] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.107 [2019-12-04 11:35:51,927] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.107 with username:root [2019-12-04 11:35:52,097] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.107 [2019-12-04 11:35:53,779] - [remote_util] [140432460064512] - INFO - 172.23.104.107 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:35:54,148] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:35:54,149] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.107: systemctl stop couchbase-server.service [2019-12-04 11:35:56,627] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:56,628] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:35:56,693] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.107: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:35:56,821] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:56,821] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.107: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:35:56,897] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:56,910] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.107 with username:root [2019-12-04 11:35:57,083] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.107 [2019-12-04 11:35:58,740] - [remote_util] [140432460064512] - INFO - 172.23.104.107 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:35:59,115] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:35:59,115] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.107: systemctl start couchbase-server.service [2019-12-04 11:35:59,240] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:35:59,240] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:35:59,359] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.109 with username:root [2019-12-04 11:35:59,528] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.109 [2019-12-04 11:36:01,163] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.109 with username:root [2019-12-04 11:36:01,334] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.109 [2019-12-04 11:36:02,985] - [remote_util] [140432460064512] - INFO - 172.23.104.109 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:36:03,427] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:36:03,430] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.109: systemctl stop couchbase-server.service [2019-12-04 11:36:05,812] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:05,813] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:36:05,929] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.109: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:36:06,065] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:06,066] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.109: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:36:06,200] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:06,209] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.109 with username:root [2019-12-04 11:36:06,379] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.109 [2019-12-04 11:36:08,314] - [remote_util] [140432460064512] - INFO - 172.23.104.109 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:36:08,733] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:36:08,734] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.109: systemctl start couchbase-server.service [2019-12-04 11:36:08,861] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:08,861] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:36:08,944] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.199 with username:root [2019-12-04 11:36:09,115] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.199 [2019-12-04 11:36:10,110] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.199 with username:root [2019-12-04 11:36:10,279] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.199 [2019-12-04 11:36:11,365] - [remote_util] [140432460064512] - INFO - 172.23.104.199 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:36:11,607] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:36:11,608] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.199: systemctl stop couchbase-server.service [2019-12-04 11:36:13,988] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:13,988] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:36:14,103] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.199: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:36:14,141] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:14,143] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.199: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:36:14,244] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:14,254] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.199 with username:root [2019-12-04 11:36:14,426] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.199 [2019-12-04 11:36:15,501] - [remote_util] [140432460064512] - INFO - 172.23.104.199 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:36:15,687] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:36:15,688] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.199: systemctl start couchbase-server.service [2019-12-04 11:36:15,756] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:15,757] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:36:15,880] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.201 with username:root [2019-12-04 11:36:16,050] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.201 [2019-12-04 11:36:17,083] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.201 with username:root [2019-12-04 11:36:17,253] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.201 [2019-12-04 11:36:18,296] - [remote_util] [140432460064512] - INFO - 172.23.104.201 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:36:18,483] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:36:18,483] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.201: systemctl stop couchbase-server.service [2019-12-04 11:36:20,560] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:20,562] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:36:20,679] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.201: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:36:20,719] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:20,719] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.201: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:36:20,790] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:20,797] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.201 with username:root [2019-12-04 11:36:20,966] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.201 [2019-12-04 11:36:21,970] - [remote_util] [140432460064512] - INFO - 172.23.104.201 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:36:22,298] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:36:22,298] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.201: systemctl start couchbase-server.service [2019-12-04 11:36:22,368] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:22,369] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:36:22,493] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.202 with username:root [2019-12-04 11:36:22,662] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.202 [2019-12-04 11:36:23,710] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.202 with username:root [2019-12-04 11:36:23,878] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.202 [2019-12-04 11:36:24,882] - [remote_util] [140432460064512] - INFO - 172.23.104.202 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:36:25,139] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:36:25,140] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.202: systemctl stop couchbase-server.service [2019-12-04 11:36:27,322] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:27,322] - [basetestcase] [140432460064512] - INFO - Couchbase stopped [2019-12-04 11:36:27,437] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.202: rm -rf /opt/couchbase/var/lib/couchbase/data/* [2019-12-04 11:36:27,475] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:27,477] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.202: rm -rf /opt/couchbase/var/lib/couchbase/config/* [2019-12-04 11:36:27,546] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:27,555] - [remote_util] [140432460064512] - INFO - connecting to 172.23.104.202 with username:root [2019-12-04 11:36:27,724] - [remote_util] [140432460064512] - INFO - Connected to 172.23.104.202 [2019-12-04 11:36:28,814] - [remote_util] [140432460064512] - INFO - 172.23.104.202 **** The version file /opt/couchbase/ VERSION.txt exists [2019-12-04 11:36:29,053] - [remote_util] [140432460064512] - INFO - Running systemd command on this server [2019-12-04 11:36:29,054] - [remote_util] [140432460064512] - INFO - running command.raw on 172.23.104.202: systemctl start couchbase-server.service [2019-12-04 11:36:29,093] - [remote_util] [140432460064512] - INFO - command executed successfully [2019-12-04 11:36:29,094] - [basetestcase] [140432460064512] - INFO - Couchbase started [2019-12-04 11:36:29,210] - [basetestcase] [140432460064512] - INFO - sleep for 10 secs. ... [2019-12-04 11:36:39,300] - [remote_util] [140432695666496] - INFO - connecting to 172.23.104.100 with username:root [2019-12-04 11:36:39,506] - [remote_util] [140432695666496] - INFO - Connected to 172.23.104.100 [2019-12-04 11:36:40,306] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.100: /opt/couchbase/bin/cbcollect_info 172.23.104.100-20191204-1136-diag.zip [2019-12-04 11:46:59,403] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:48:24,731] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.100: rm -f /root/172.23.104.100-20191204-1136-diag.zip [2019-12-04 11:48:25,117] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:48:25,242] - [remote_util] [140432695666496] - INFO - connecting to 172.23.104.103 with username:root [2019-12-04 11:48:25,379] - [remote_util] [140432695666496] - INFO - Connected to 172.23.104.103 [2019-12-04 11:48:26,258] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.103: /opt/couchbase/bin/cbcollect_info 172.23.104.103-20191204-1148-diag.zip [2019-12-04 11:50:05,233] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:50:10,228] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.103: rm -f /root/172.23.104.103-20191204-1148-diag.zip [2019-12-04 11:50:10,272] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:50:10,394] - [remote_util] [140432695666496] - INFO - connecting to 172.23.104.105 with username:root [2019-12-04 11:50:10,531] - [remote_util] [140432695666496] - INFO - Connected to 172.23.104.105 [2019-12-04 11:50:11,354] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.105: /opt/couchbase/bin/cbcollect_info 172.23.104.105-20191204-1150-diag.zip [2019-12-04 11:51:34,855] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:51:36,484] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.105: rm -f /root/172.23.104.105-20191204-1150-diag.zip [2019-12-04 11:51:36,568] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:51:36,689] - [remote_util] [140432695666496] - INFO - connecting to 172.23.104.107 with username:root [2019-12-04 11:51:36,858] - [remote_util] [140432695666496] - INFO - Connected to 172.23.104.107 [2019-12-04 11:51:38,396] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.107: /opt/couchbase/bin/cbcollect_info 172.23.104.107-20191204-1151-diag.zip [2019-12-04 11:53:21,082] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:53:22,480] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.107: rm -f /root/172.23.104.107-20191204-1151-diag.zip [2019-12-04 11:53:22,614] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:53:22,734] - [remote_util] [140432695666496] - INFO - connecting to 172.23.104.109 with username:root [2019-12-04 11:53:22,904] - [remote_util] [140432695666496] - INFO - Connected to 172.23.104.109 [2019-12-04 11:53:24,444] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.109: /opt/couchbase/bin/cbcollect_info 172.23.104.109-20191204-1153-diag.zip [2019-12-04 11:55:04,918] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:55:06,371] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.109: rm -f /root/172.23.104.109-20191204-1153-diag.zip [2019-12-04 11:55:06,559] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:55:06,628] - [remote_util] [140432695666496] - INFO - connecting to 172.23.104.199 with username:root [2019-12-04 11:55:06,798] - [remote_util] [140432695666496] - INFO - Connected to 172.23.104.199 [2019-12-04 11:55:07,715] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.199: /opt/couchbase/bin/cbcollect_info 172.23.104.199-20191204-1155-diag.zip [2019-12-04 11:56:56,497] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:56:57,996] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.199: rm -f /root/172.23.104.199-20191204-1155-diag.zip [2019-12-04 11:56:58,100] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:56:58,224] - [remote_util] [140432695666496] - INFO - connecting to 172.23.104.201 with username:root [2019-12-04 11:56:58,395] - [remote_util] [140432695666496] - INFO - Connected to 172.23.104.201 [2019-12-04 11:56:59,423] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.201: /opt/couchbase/bin/cbcollect_info 172.23.104.201-20191204-1156-diag.zip [2019-12-04 11:58:39,552] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:58:41,347] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.201: rm -f /root/172.23.104.201-20191204-1156-diag.zip [2019-12-04 11:58:41,450] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 11:58:41,573] - [remote_util] [140432695666496] - INFO - connecting to 172.23.104.202 with username:root [2019-12-04 11:58:41,746] - [remote_util] [140432695666496] - INFO - Connected to 172.23.104.202 [2019-12-04 11:58:42,814] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.202: /opt/couchbase/bin/cbcollect_info 172.23.104.202-20191204-1158-diag.zip [2019-12-04 12:00:26,467] - [remote_util] [140432695666496] - INFO - command executed successfully [2019-12-04 12:00:27,234] - [remote_util] [140432695666496] - INFO - running command.raw on 172.23.104.202: rm -f /root/172.23.104.202-20191204-1158-diag.zip [2019-12-04 12:00:27,281] - [remote_util] [140432695666496] - INFO - command executed successfully