SyncWrites requiring persistence may get stuck during rebalance if persisted while vBucket is dead

>>> 

>>> r = cb_coll.upsert("test_docs-4991", "")

Traceback (most recent call last):

  File "<stdin>", line 1, in <module>

  File "/Users/ritesh.agarwal/Library/Python/3.8/lib/python/site-packages/couchbase/collection.py", line 293, in wrapped

    return func(self, *args, **kwargs)

  File "/Users/ritesh.agarwal/Library/Python/3.8/lib/python/site-packages/couchbase/result.py", line 682, in mutated

    result = func(*args, **kwargs)

  File "/Users/ritesh.agarwal/Library/Python/3.8/lib/python/site-packages/couchbase/collection.py", line 878, in upsert

    return ResultPrecursor(CoreClient.upsert(

couchbase.exceptions.CouchbaseException: <Key='test_docs-4991', RC=0x136[LCB_ERR_DURABLE_WRITE_IN_PROGRESS (310)], Operational Error, Results=1, C Source=(src/multiresult.c,332), Context={'status_code': 4, 'opaque': 3, 'cas': 0, 'key': 'test_docs-4991', 'bucket': 'default', 'collection': '_default', 'scope': '_default', 'context': '', 'ref': '', 'endpoint': '172.23.121.116:11210', 'type': 'KVErrorContext'}, Tracing Output={"test_docs-4991": {"debug_info": {"FILE": "src/callbacks.c", "FUNC": "dur_chain2", "LINE": 751}}}>

>>> 

>>> 

>>> r = cb_coll.get("test_docs-4991")

Traceback (most recent call last):

  File "<stdin>", line 1, in <module>

  File "/Users/ritesh.agarwal/Library/Python/3.8/lib/python/site-packages/couchbase/collection.py", line 293, in wrapped

    return func(self, *args, **kwargs)

  File "/Users/ritesh.agarwal/Library/Python/3.8/lib/python/site-packages/couchbase/result.py", line 544, in wrapped

    x, options = func(*args, **kwargs)

  File "/Users/ritesh.agarwal/Library/Python/3.8/lib/python/site-packages/couchbase/collection.py", line 521, in get

    return self._get_generic(key, kwargs, options)

  File "/Users/ritesh.agarwal/Library/Python/3.8/lib/python/site-packages/couchbase/collection.py", line 484, in _get_generic

    x = CoreClient.get(self.bucket, key, **opts)

  File "/Users/ritesh.agarwal/Library/Python/3.8/lib/python/site-packages/couchbase_core/client.py", line 409, in get

    return super(Client, self).get(*args, **kwargs)

couchbase.exceptions.CouchbaseException: <Key='test_docs-4991', RC=0x137[LCB_ERR_DURABLE_WRITE_RE_COMMIT_IN_PROGRESS (311)], Operational Error, Results=1, C Source=(src/multiresult.c,332), Context={'status_code': 164, 'opaque': 7, 'cas': 0, 'key': 'test_docs-4991', 'bucket': 'default', 'collection': '_default', 'scope': '_default', 'context': '', 'ref': '', 'endpoint': '172.23.121.116:11210', 'type': 'KVErrorContext'}, Tracing Output={"test_docs-4991": {"debug_info": {"FILE": "src/callbacks.c", "FUNC": "value_callback", "LINE": 856}}}>

>>> 

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/magma_temp_job4.ini -p GROUP=P0;durability,durability=MAJORITY_AND_PERSIST_TO_ACTIVE,bucket_storage=magma,rerun=False,enable_dp=false,autoCompactionDefined=true,infra_log_level=debug,log_level=debug,bucket_storage=magma,upgrade_version=7.1.0-2506 -t rebalance_new.rebalance_in_out.RebalanceInOutTests.test_incremental_rebalance_out_in_with_mutation,upgrade_version=7.1.0-2506,rerun=False,enable_dp=false,GROUP=P0;durability,get-cbcollect-info=False,replicas=1,durability=MAJORITY_AND_PERSIST_TO_ACTIVE,bucket_storage=magma,log_level=debug,nodes_init=3,autoCompactionDefined=true,infra_log_level=debug'