[FTS] Partial results not returned when fts is killed on one of the nodes

[2016-05-01 22:35:38,953] - [remote_util:2234] INFO - running command.raw on 172.23.105.190: killall -9 cbft

[2016-05-01 22:35:39,053] - [remote_util:2271] INFO - command executed successfully

[2016-05-01 22:35:39,771] - [task:1098] INFO - ------------------------------------------------------------------ Query # 1 -----------------------------------------------------------------

[2016-05-01 22:35:39,792] - [fts_base:1104] INFO - Running query {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "manages.reports", "match": "Hanna Drucilla"}, "size": 10000000} on node: 172.23.106.66:

[2016-05-01 22:35:39,857] - [rest_client:775] ERROR - http://172.23.106.66:8094/api/index/default_index/query error 400 reason: status: 400, content: rest_index: Query, indexName: default_index, requestBody: {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "manages.reports", "match": "Hanna Drucilla"}, "size": 10000000}, req: &http.Request{Method:"POST", URL:(*url.URL)(0xc8207e7a00), Proto:"HTTP/1.1", ProtoMajor:1, ProtoMinor:1, Header:http.Header{"Accept-Encoding":[]string{"identity"}, "Content-Length":[]string{"229"}, "Content-Type":[]string{"application/json"}, "Authorization":[]string{"Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA=="}, "Accept":[]string{"*/*"}, "User-Agent":[]string{"Python-httplib2/$Rev: 259 $"}}, Body:(*http.body)(0xc82162a2c0), ContentLength:229, TransferEncoding:[]string(nil), Close:false, Host:"172.23.106.66:8094", Form:url.Values{}, PostForm:url.Values{}, MultipartForm:(*multipart.Form)(nil), Trailer:http.Header(nil), RemoteAddr:"172.23.122.47:55417", RequestURI:"/api/index/default_index/query", TLS:(*tls.ConnectionState)(nil), Cancel:(<-chan struct {})(nil)}, err: bleve: QueryBlevePIndexImpl remote client returned status: 403 body: rest_auth: preparePerm, err: pindex not found 

 rest_index: Query, indexName: default_index, requestBody: {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "manages.reports", "match": "Hanna Drucilla"}, "size": 10000000}, req: &http.Request{Method:"POST", URL:(*url.URL)(0xc8207e7a00), Proto:"HTTP/1.1", ProtoMajor:1, ProtoMinor:1, Header:http.Header{"Accept-Encoding":[]string{"identity"}, "Content-Length":[]string{"229"}, "Content-Type":[]string{"application/json"}, "Authorization":[]string{"Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA=="}, "Accept":[]string{"*/*"}, "User-Agent":[]string{"Python-httplib2/$Rev: 259 $"}}, Body:(*http.body)(0xc82162a2c0), ContentLength:229, TransferEncoding:[]string(nil), Close:false, Host:"172.23.106.66:8094", Form:url.Values{}, PostForm:url.Values{}, MultipartForm:(*multipart.Form)(nil), Trailer:http.Header(nil), RemoteAddr:"172.23.122.47:55417", RequestURI:"/api/index/default_index/query", TLS:(*tls.ConnectionState)(nil), Cancel:(<-chan struct {})(nil)}, err: bleve: QueryBlevePIndexImpl remote client returned status: 403 body: rest_auth: preparePerm, err: pindex not found 

[2016-05-01 22:35:39,858] - [fts_base:683] ERROR - Error running query: 'NoneType' object is not iterable

[2016-05-01 22:35:39,858] - [task:1098] INFO - ------------------------------------------------------------------ Query # 2 -----------------------------------------------------------------

[2016-05-01 22:35:39,873] - [fts_base:1104] INFO - Running query {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "manages.reports", "match": "Solita Hanna Cytheria"}, "size": 10000000} on node: 172.23.106.66:

[2016-05-01 22:35:39,923] - [rest_client:775] ERROR - http://172.23.106.66:8094/api/index/default_index/query error 400 reason: status: 400, content: rest_index: Query, indexName: default_index, requestBody: {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "manages.reports", "match": "Solita Hanna Cytheria"}, "size": 10000000}, req: &http.Request{Method:"POST", URL:(*url.URL)(0xc8207e6c00), Proto:"HTTP/1.1", ProtoMajor:1, ProtoMinor:1, Header:http.Header{"Content-Type":[]string{"application/json"}, "Authorization":[]string{"Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA=="}, "Accept":[]string{"*/*"}, "User-Agent":[]string{"Python-httplib2/$Rev: 259 $"}, "Accept-Encoding":[]string{"identity"}, "Content-Length":[]string{"236"}}, Body:(*http.body)(0xc8210c6d40), ContentLength:236, TransferEncoding:[]string(nil), Close:false, Host:"172.23.106.66:8094", Form:url.Values{}, PostForm:url.Values{}, MultipartForm:(*multipart.Form)(nil), Trailer:http.Header(nil), RemoteAddr:"172.23.122.47:55421", RequestURI:"/api/index/default_index/query", TLS:(*tls.ConnectionState)(nil), Cancel:(<-chan struct {})(nil)}, err: bleve: QueryBlevePIndexImpl remote client returned status: 403 body: rest_auth: preparePerm, err: pindex not found 

 rest_index: Query, indexName: default_index, requestBody: {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "manages.reports", "match": "Solita Hanna Cytheria"}, "size": 10000000}, req: &http.Request{Method:"POST", URL:(*url.URL)(0xc8207e6c00), Proto:"HTTP/1.1", ProtoMajor:1, ProtoMinor:1, Header:http.Header{"Content-Type":[]string{"application/json"}, "Authorization":[]string{"Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA=="}, "Accept":[]string{"*/*"}, "User-Agent":[]string{"Python-httplib2/$Rev: 259 $"}, "Accept-Encoding":[]string{"identity"}, "Content-Length":[]string{"236"}}, Body:(*http.body)(0xc8210c6d40), ContentLength:236, TransferEncoding:[]string(nil), Close:false, Host:"172.23.106.66:8094", Form:url.Values{}, PostForm:url.Values{}, MultipartForm:(*multipart.Form)(nil), Trailer:http.Header(nil), RemoteAddr:"172.23.122.47:55421", RequestURI:"/api/index/default_index/query", TLS:(*tls.ConnectionState)(nil), Cancel:(<-chan struct {})(nil)}, err: bleve: QueryBlevePIndexImpl remote client returned status: 403 body: rest_auth: preparePerm, err: pindex not found 

[2016-05-01 22:35:39,923] - [fts_base:683] ERROR - Error running query: 'NoneType' object is not iterable

[2016-05-01 22:35:39,923] - [task:1098] INFO - ------------------------------------------------------------------ Query # 3 -----------------------------------------------------------------

[2016-05-01 22:35:39,937] - [fts_base:1104] INFO - Running query {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "dept", "match": "Training"}, "size": 10000000} on node: 172.23.106.66:

[2016-05-01 22:35:39,992] - [rest_client:775] ERROR - http://172.23.106.66:8094/api/index/default_index/query error 400 reason: status: 400, content: rest_index: Query, indexName: default_index, requestBody: {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "dept", "match": "Training"}, "size": 10000000}, req: &http.Request{Method:"POST", URL:(*url.URL)(0xc820292980), Proto:"HTTP/1.1", ProtoMajor:1, ProtoMinor:1, Header:http.Header{"Content-Type":[]string{"application/json"}, "Authorization":[]string{"Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA=="}, "Accept":[]string{"*/*"}, "User-Agent":[]string{"Python-httplib2/$Rev: 259 $"}, "Accept-Encoding":[]string{"identity"}, "Content-Length":[]string{"212"}}, Body:(*http.body)(0xc8212e60c0), ContentLength:212, TransferEncoding:[]string(nil), Close:false, Host:"172.23.106.66:8094", Form:url.Values{}, PostForm:url.Values{}, MultipartForm:(*multipart.Form)(nil), Trailer:http.Header(nil), RemoteAddr:"172.23.122.47:55425", RequestURI:"/api/index/default_index/query", TLS:(*tls.ConnectionState)(nil), Cancel:(<-chan struct {})(nil)}, err: bleve: QueryBlevePIndexImpl remote client returned status: 403 body: rest_auth: preparePerm, err: pindex not found 

 rest_index: Query, indexName: default_index, requestBody: {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "dept", "match": "Training"}, "size": 10000000}, req: &http.Request{Method:"POST", URL:(*url.URL)(0xc820292980), Proto:"HTTP/1.1", ProtoMajor:1, ProtoMinor:1, Header:http.Header{"Content-Type":[]string{"application/json"}, "Authorization":[]string{"Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA=="}, "Accept":[]string{"*/*"}, "User-Agent":[]string{"Python-httplib2/$Rev: 259 $"}, "Accept-Encoding":[]string{"identity"}, "Content-Length":[]string{"212"}}, Body:(*http.body)(0xc8212e60c0), ContentLength:212, TransferEncoding:[]string(nil), Close:false, Host:"172.23.106.66:8094", Form:url.Values{}, PostForm:url.Values{}, MultipartForm:(*multipart.Form)(nil), Trailer:http.Header(nil), RemoteAddr:"172.23.122.47:55425", RequestURI:"/api/index/default_index/query", TLS:(*tls.ConnectionState)(nil), Cancel:(<-chan struct {})(nil)}, err: bleve: QueryBlevePIndexImpl remote client returned status: 403 body: rest_auth: preparePerm, err: pindex not found 

[2016-05-01 22:35:39,993] - [fts_base:683] ERROR - Error running query: 'NoneType' object is not iterable

[2016-05-01 22:35:39,993] - [task:1098] INFO - ------------------------------------------------------------------ Query # 4 -----------------------------------------------------------------

[2016-05-01 22:35:40,008] - [fts_base:1104] INFO - Running query {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "type", "match": "wiki"}, "size": 10000000} on node: 172.23.106.66:

[2016-05-01 22:35:40,043] - [rest_client:775] ERROR - http://172.23.106.66:8094/api/index/default_index/query error 400 reason: status: 400, content: rest_index: Query, indexName: default_index, requestBody: {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "type", "match": "wiki"}, "size": 10000000}, req: &http.Request{Method:"POST", URL:(*url.URL)(0xc8207e6300), Proto:"HTTP/1.1", ProtoMajor:1, ProtoMinor:1, Header:http.Header{"Content-Length":[]string{"208"}, "Content-Type":[]string{"application/json"}, "Authorization":[]string{"Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA=="}, "Accept":[]string{"*/*"}, "User-Agent":[]string{"Python-httplib2/$Rev: 259 $"}, "Accept-Encoding":[]string{"identity"}}, Body:(*http.body)(0xc8218c3a40), ContentLength:208, TransferEncoding:[]string(nil), Close:false, Host:"172.23.106.66:8094", Form:url.Values{}, PostForm:url.Values{}, MultipartForm:(*multipart.Form)(nil), Trailer:http.Header(nil), RemoteAddr:"172.23.122.47:55429", RequestURI:"/api/index/default_index/query", TLS:(*tls.ConnectionState)(nil), Cancel:(<-chan struct {})(nil)}, err: bleve: QueryBlevePIndexImpl remote client returned status: 403 body: rest_auth: preparePerm, err: pindex not found 

 rest_index: Query, indexName: default_index, requestBody: {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "type", "match": "wiki"}, "size": 10000000}, req: &http.Request{Method:"POST", URL:(*url.URL)(0xc8207e6300), Proto:"HTTP/1.1", ProtoMajor:1, ProtoMinor:1, Header:http.Header{"Content-Length":[]string{"208"}, "Content-Type":[]string{"application/json"}, "Authorization":[]string{"Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA=="}, "Accept":[]string{"*/*"}, "User-Agent":[]string{"Python-httplib2/$Rev: 259 $"}, "Accept-Encoding":[]string{"identity"}}, Body:(*http.body)(0xc8218c3a40), ContentLength:208, TransferEncoding:[]string(nil), Close:false, Host:"172.23.106.66:8094", Form:url.Values{}, PostForm:url.Values{}, MultipartForm:(*multipart.Form)(nil), Trailer:http.Header(nil), RemoteAddr:"172.23.122.47:55429", RequestURI:"/api/index/default_index/query", TLS:(*tls.ConnectionState)(nil), Cancel:(<-chan struct {})(nil)}, err: bleve: QueryBlevePIndexImpl remote client returned status: 403 body: rest_auth: preparePerm, err: pindex not found 

[2016-05-01 22:35:40,044] - [fts_base:683] ERROR - Error running query: 'NoneType' object is not iterable

[2016-05-01 22:35:40,044] - [task:1098] INFO - ------------------------------------------------------------------ Query # 5 -----------------------------------------------------------------

[2016-05-01 22:35:40,067] - [fts_base:1104] INFO - Running query {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "email", "match": "fuscienne@mcdiabetes.com"}, "size": 10000000} on node: 172.23.106.66:

[2016-05-01 22:35:40,231] - [rest_client:775] ERROR - http://172.23.106.66:8094/api/index/default_index/query error 400 reason: status: 400, content: rest_index: Query, indexName: default_index, requestBody: {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "email", "match": "fuscienne@mcdiabetes.com"}, "size": 10000000}, req: &http.Request{Method:"POST", URL:(*url.URL)(0xc8207e6580), Proto:"HTTP/1.1", ProtoMajor:1, ProtoMinor:1, Header:http.Header{"Accept-Encoding":[]string{"identity"}, "Content-Length":[]string{"229"}, "Content-Type":[]string{"application/json"}, "Authorization":[]string{"Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA=="}, "Accept":[]string{"*/*"}, "User-Agent":[]string{"Python-httplib2/$Rev: 259 $"}}, Body:(*http.body)(0xc821062740), ContentLength:229, TransferEncoding:[]string(nil), Close:false, Host:"172.23.106.66:8094", Form:url.Values{}, PostForm:url.Values{}, MultipartForm:(*multipart.Form)(nil), Trailer:http.Header(nil), RemoteAddr:"172.23.122.47:55433", RequestURI:"/api/index/default_index/query", TLS:(*tls.ConnectionState)(nil), Cancel:(<-chan struct {})(nil)}, err: bleve: QueryBlevePIndexImpl remote client returned status: 403 body: rest_auth: preparePerm, err: pindex not found 

 rest_index: Query, indexName: default_index, requestBody: {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "email", "match": "fuscienne@mcdiabetes.com"}, "size": 10000000}, req: &http.Request{Method:"POST", URL:(*url.URL)(0xc8207e6580), Proto:"HTTP/1.1", ProtoMajor:1, ProtoMinor:1, Header:http.Header{"Accept-Encoding":[]string{"identity"}, "Content-Length":[]string{"229"}, "Content-Type":[]string{"application/json"}, "Authorization":[]string{"Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA=="}, "Accept":[]string{"*/*"}, "User-Agent":[]string{"Python-httplib2/$Rev: 259 $"}}, Body:(*http.body)(0xc821062740), ContentLength:229, TransferEncoding:[]string(nil), Close:false, Host:"172.23.106.66:8094", Form:url.Values{}, PostForm:url.Values{}, MultipartForm:(*multipart.Form)(nil), Trailer:http.Header(nil), RemoteAddr:"172.23.122.47:55433", RequestURI:"/api/index/default_index/query", TLS:(*tls.ConnectionState)(nil), Cancel:(<-chan struct {})(nil)}, err: bleve: QueryBlevePIndexImpl remote client returned status: 403 body: rest_auth: preparePerm, err: pindex not found 

[2016-05-01 22:35:40,231] - [fts_base:683] ERROR - Error running query: 'NoneType' object is not iterable

[2016-05-01 22:35:40,232] - [task:1098] INFO - ------------------------------------------------------------------ Query # 6 -----------------------------------------------------------------

[2016-05-01 22:35:40,252] - [fts_base:1104] INFO - Running query {"from": 0, "indexName": "default_index", "fields": [], "explain": false, "ctl": {"timeout": 60000, "consistency": {"vectors": {}, "level": ""}}, "query": {"field": "languages_known", "match": "French"}, "size": 10000000} on node: 172.23.106.66:

[2016-05-01 22:35:40,682] - [rest_client:2304] INFO - Status: {u'successful': 32, u'failed': 0, u'total': 32}

[2016-05-01 22:35:40,703] - [task:1107] INFO - FTS hits for query: {"field": "languages_known", "match": "French"} is 7922 (took 141.215573ms)