Data mismatch with rebalance + timers + curl

 function OnUpdate(doc, meta) {

    var expiry = new Date();

    expiry.setSeconds(expiry.getSeconds() + 30);

    var context = {docID : meta.id, random_text : "e6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh07Aumoe6cZZGHuh07Aumoe6cZZGHuh07Aumoe6"};

    createTimer(timerCallback,  expiry, meta.id, context);

}

function OnDelete(meta) {

    var expiry = new Date();

    expiry.setSeconds(expiry.getSeconds() + 30);

    var context = {docID : meta.id};

    createTimer(NDtimerCallback,  expiry, meta.id, context);

}

function NDtimerCallback(context) {

    var request = {

	path : 'post?param=text',

	headers: {

    "cache-control": "no-cache",

    "Postman-Token": "a3e931fe-8fe2-413c-be82-546062d28377"

    },

    body: "This is expected to be sent back as part of response body."

    };

    try {

    	var response = curl("POST", server, request);

    	log('response body received from server:', response.body);

    	log('response headers received from server:', response.headers);

    	log('response status received from server:', response.status);

    	var res= new Uint8Array(response.body);

    	if(response.status == 200){

    	    delete dst_bucket[context.docID];

    	}

    	else{

    	    delete dst_bucket[context.docID];

    	}

    }

    catch (e) {

    	log('error:', e);

        }

}

function timerCallback(context) {

    var request = {

	path : 'post?param=text',

	headers: {

    "cache-control": "no-cache",

    "Postman-Token": "a3e931fe-8fe2-413c-be82-546062d28377"

    },

    body: "This is expected to be sent back as part of response body."

    };

    try {

    	var response = curl("POST", server, request);

    	log('response body received from server:', response.body);

    	log('response headers received from server:', response.headers);

    	log('response status received from server:', response.status);

    	var res= new Uint8Array(response.body);

    	if(response.status == 200){

    	    dst_bucket[context.docID]=response.body;

    	}

    	else{

    	    dst_bucket[context.docID]=response.status;

    	}

    }

    catch (e) {

    	log('error:', e);

        }

}

/testrunner -i /tmp/testexec.11548.ini -p get-cbcollect-info=True,GROUP=timer_op_curl -t eventing.eventing_rebalance.EventingRebalance.test_eventing_rebalance_out_when_existing_eventing_node_is_processing_mutations,nodes_init=5,services_init=kv-eventing-eventing-index-n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=20,handler_code=timer_op_curl_post,curl=True,GROUP=timer_op_curl;timer_op_curl_windows

 Exception: duplicated data in destination bucket which is not expected. Current : 2 Expected : 0  dcp_backlog : 0  TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.105.122': {}}

./testrunner -i /tmp/testexec.11548.ini -p get-cbcollect-info=True,GROUP=timer_op_curl -t eventing.eventing_rebalance.EventingRebalance.test_eventing_swap_rebalance_when_existing_eventing_node_is_processing_mutations,nodes_init=5,services_init=kv-eventing-eventing-index-n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=20,handler_code=timer_op_curl_put,curl=True,GROUP=timer_op_curl;timer_op_curl_windows

Exception: missing data in destination bucket. Current : 40318 Expected : 40320  dcp_backlog : 0  TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.105.185': {}} 

./testrunner -i /tmp/testexec.11548.ini -p get-cbcollect-info=True,GROUP=timer_op_curl -t eventing.eventing_rebalance.EventingRebalance.test_kv_rebalance_out_when_existing_eventing_node_is_processing_mutations,nodes_init=7,services_init=kv-kv-kv-eventing-eventing-eventing-index:n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=20,handler_code=timer_op_curl_post,curl=True,GROUP=timer_op_curl;timer_op_curl_windows

Exception: duplicated data in destination bucket which is not expected. Current : 1 Expected : 0  dcp_backlog : 0  TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.105.163': {}, '172.23.105.137': {}, '172.23.105.185': {}} 

./testrunner -i /tmp/testexec.11548.ini -p get-cbcollect-info=True,GROUP=timer_op_curl -t eventing.eventing_rebalance.EventingRebalance.test_kv_swap_rebalance_when_existing_eventing_node_is_processing_mutations,nodes_init=6,services_init=kv-kv-eventing-eventing-eventing-index:n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=20,handler_code=timer_op_curl_put,curl=True,GROUP=timer_op_curl;timer_op_curl_windows

Exception: duplicated data in destination bucket which is not expected. Current : 3 Expected : 0  dcp_backlog : 0  TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.105.163': {}, '172.23.105.130': {}, '172.23.105.137': {}} 

./testrunner -i /tmp/testexec.11548.ini -p get-cbcollect-info=True,GROUP=timer_op_curl -t eventing.eventing_rebalance.EventingRebalance.test_eventing_rebalance_with_multiple_eventing_nodes,doc-per-day=5,dataset=default,nodes_init=5,services_init=kv-eventing-eventing-eventing-index:n1ql,groups=simple,reset_services=True,handler_code=timer_op_curl_delete,curl=True,GROUP=timer_op_curl

Exception: missing data in destination bucket. Current : 10055 Expected : 10080  dcp_backlog : 0  TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.105.188': {}, '172.23.105.130': {}, '172.23.105.137': {}, '172.23.105.122': {}, '172.23.105.185': {}}

./testrunner -i /tmp/testexec.11548.ini -p get-cbcollect-info=True,GROUP=timer_op_curl -t eventing.eventing_rebalance.EventingRebalance.test_eventing_rebalance_with_multiple_kv_nodes,doc-per-day=5,dataset=default,nodes_init=5,services_init=kv-kv-kv-eventing-index:n1ql,groups=simple,reset_services=True,handler_code=timer_op_curl_get,curl=True,GROUP=timer_op_curl

Exception: missing data in destination bucket. Current : 10078 Expected : 10080  dcp_backlog : 0  TIMERS_IN_PAST : 0 lcb_exceptions : {'172.23.105.137': {}}