Mismatch on expected geospatial view results on failover tests

1. Create 4 node cluster, with 2 bucket, create spatial views
2. Load 10K items
3. Issue simple limit queries
4. Failover 1 node ( failover_factor=1)
5. Rebalance out the failed over node
6. Keep continously querying the dataset
7. After rebalance is complete - check num-expected documents

• mismatch on expected documents with below errors from ns_server

test to repro:
./testrunner -i /tmp/geo_vm.ini -t view.spatialquerytests.SpatialQueryTests.test_failover_simple_dataset_limit_queries,num-docs=10000,failover_factor=1

"errors": [{"from": "10.6.2.226:8092", "reason": "{badarg,[

{erlang,list_to_integer,[\"HTTP/1.1500InternalServerError\",16],[]},\n {lhttpc_client,chunk_size,1,\n [{file,\"/home/couchbase/jenkins/workspace/sherlock-unix/couchdb/src/lhttpc/lhttpc_client.erl\"},\n {line,736}]},\n {lhttpc_client,read_partial_chunked_body,6,\n [{file,"/home/couchbase/jenkins/workspace/sherlock-unix/couchdb/src/lhttpc/lhttpc_client.erl"},\n {line,649}]},\n {lhttpc_client,handle_response_body,4,\n [{file,\"/home/couchbase/jenkins/workspace/sherlock-unix/couchdb/src/lhttpc/lhttpc_client.erl\"},\n {line,497}]},\n {lhttpc_client,read_response,4,\n [{file,"/home/couchbase/jenkins/workspace/sherlock-unix/couchdb/src/lhttpc/lhttpc_client.erl"},\n {line,446}]},\n {lhttpc_client,execute,9,\n [{file,\"/home/couchbase/jenkins/workspace/sherlock-unix/couchdb/src/lhttpc/lhttpc_client.erl\"},\n {line,169}]},\n {lhttpc_client,request,9,\n [{file,"/home/couchbase/jenkins/workspace/sherlock-unix/couchdb/src/lhttpc/lhttpc_client.erl"},\n {line,92}]}]}"}, {"from": "10.6.2.227:8092", "reason": "{badarg,[{erlang,list_to_integer,["HTTP/1.1500InternalServerError",16],[]}

,\n {lhttpc_client,chunk_size,1,\n [

{file,\"/home/couchbase/jenkins/workspace/sherlock-unix/couchdb/src/lhttpc/lhttpc_client.erl\"},\n {line,736}]},\n {lhttpc_client,read_partial_chunked_body,6,\n [{file,"/home/couchbase/jenkins/workspace/sherlock-unix/couchdb/src/lhttpc/lhttpc_client.erl"}

,\n

{line,649}

]},\n {lhttpc_client,handle_response_body,4,\n [

{file,\"/home/couchbase/jenkins/workspace/sherlock-unix/couchdb/src/lhttpc/lhttpc_client.erl\"},\n {line,497}]},\n {lhttpc_client,read_response,4,\n [{file,"/home/couchbase/jenkins/workspace/sherlock-unix/couchdb/src/lhttpc/lhttpc_client.erl"}

,\n

{line,446}

]},\n {lhttpc_client,execute,9,\n [

{file,\"/home/couchbase/jenkins/workspace/sherlock-unix/couchdb/src/lhttpc/lhttpc_client.erl\"},\n {line,169}]},\n {lhttpc_client,request,9,\n [{file,"/home/couchbase/jenkins/workspace/sherlock-unix/couchdb/src/lhttpc/lhttpc_client.erl"}

,\n

{line,92}

]}]}"}], "rows": [{"geometry":

{"coordinates": [-142, -70], "type": "Point"}

, "id": "simple_dataset-9177", "key": [[-142, -142], [-70, -70]], "node": "local", "partition": 285, "value": "simple_dataset-9177"}, {"geometry":

{"coordinates": [-127, -57], "type": "Point"}

, "id": "simple_dataset-8005", "key": [[-127, -127], [-57, -57]], "node": "local", "partition": 300, "value": "simple_dataset-8005"}, {"geometry":

{"coordinates": [-179, -51], "type": "Point"}

, "id": "simple_dataset-8365", "key": [[-179, -179], [-51, -51]], "node": "local", "partition": 304, "value": "simple_dataset-8365"}, {"geometry":

{"coordinates": [-150, -49], "type": "Point"}

, "id": "simple_dataset-8807", "key": [[-150, -150], [-49, -49]], "node": "local", "partition": 305, "value": "simple_dataset-8807"}, {"geometry":

{"coordinates": [-172, -71], "type": "Point"}

, "id": "simple_dataset-9547", "key": [[-172, -172], [-71, -71]], "node": "local", "partition": 313, "value": "simple_dataset-9547"}, {"geometry":

{"coordinates": [-136, -60], "type": "Point"}

, "id": "simple_dataset-9270", "key": [[-136, -136], [-60, -60]], "node": "local", "partition": 319, "value": "simple_dataset-9270"}, {"geometry":

{"coordinates": [-129, -65], "type": "Point"}

, "id": "simple_dataset-8000", "key": [[-129, -129], [-65, -65]], "node": "local", "partition": 326, "value": "simple_dataset-8000"}, {"geometry":

{"coordinates": [-115, -72], "type": "Point"}

, "id": "simple_dataset-9059", "key": [[-115, -115], [-72, -72]], "node": "local", "partition": 337, "value": "simple_dataset-9059"}, {"geometry":

{"coordinates": [-106, -67], "type": "Point"}

, "id": "simple_dataset-9507", "key": [[-106, -106], [-67, -67]], "node": "local", "partition": 341, "value": "simple_dataset-9507"}, {"geometry":

{"coordinates": [-137, -86], "type": "Point"}

, "id": "simple_dataset-9015", "key": [[-137, -137], [-86, -86]], "node": "local", "partition": 139, "value": "simple_dataset-9015"}], "total_rows": 0}

attaching logs.