Investigate and fix test_replication_multiple_sgs.py/test_multiple_sgs_with_differrent_revs_limit

Error MessageFailed: Timeout >1800sStacktraceparams_from_base_test_setup = {'base_url': 'http://10.100.150.148:8080', 'cbl_db': 'cbl-test1708040668.7395751', 'cbs_ce': False, 'cluster_config': 'resources/cluster_configs/multiple_sync_gateways_cc', ...}

setup_customized_teardown_test = {'cbl_db1': <CBLClient.MemoryPointer.MemoryPointer object at 0x10b6ce910>, 'cbl_db2': <CBLClient.MemoryPointer.MemoryP...cbl_db3': <CBLClient.MemoryPointer.MemoryPointer object at 0x10c952550>, 'cbl_db_name1': 'cbl_db11708040669.1544', ...}

sg_conf_name = 'listener_tests/multiple_sync_gateways_rev', num_of_docs = 10

    @pytest.mark.listener

    @pytest.mark.replication

    @pytest.mark.parametrize("sg_conf_name, num_of_docs", [

        pytest.param('listener_tests/multiple_sync_gateways_rev', 10, marks=pytest.mark.sanity),

        ('listener_tests/multiple_sync_gateways_rev', 100),

        ('listener_tests/multiple_sync_gateways_rev', 1000)

    ])

    def test_multiple_sgs_with_differrent_revs_limit(params_from_base_test_setup, setup_customized_teardown_test, sg_conf_name, num_of_docs):

        """

            @summary:

            1. Create 2 DBs in 2 SGS and Create docs in CBL

            2. Set up 1 SG with revs limit 30 and SG2 with revs limit 25.

            3. Do push replication to two SGS(each DB

            to each SG).

            4. Do updates on CBL for 35 times.

            5. Continue push replication to SGs

            6. Verify that revs maintained on two SGS according to revs_limit.

            7. exchange DBs of SG and do push replication.

            8. Verify that revs maintained on two SGS according to revs_limit.

        """

        sg_db1 = "sg_db1"

        sg_db2 = "sg_db2"

        protocol = "ws"

        sg_mode = params_from_base_test_setup["mode"]

        cluster_config = params_from_base_test_setup["cluster_config"]

        base_url = params_from_base_test_setup["base_url"]

        sync_gateway_version = params_from_base_test_setup["sync_gateway_version"]

        sg_ssl = params_from_base_test_setup["sg_ssl"]

        need_sgw_admin_auth = params_from_base_test_setup["need_sgw_admin_auth"]

        db = Database(base_url)

        channels1 = ["Replication1"]

        channels2 = ["Replication2"]

        name1 = "autotest1"

        name2 = "autotest2"

        sg_client = MobileRestClient()

        if sync_gateway_version < "2.0":

            pytest.skip('--no-conflicts is enabled and does not work with sg < 2.0 , so skipping the test')

        c = cluster.Cluster(cluster_config)

        sg_config = sync_gateway_config_path_for_mode(sg_conf_name, sg_mode)

        c.reset(sg_config_path=sg_config)

        sg1 = c.sync_gateways[0]

        sg2 = c.sync_gateways[1]

        # Setting revs_limit to sg1 and sg2

        revs_limit1 = 23

        revs_limit2 = 23

        admin = Admin(sg1)

        admin.admin_url = sg1.url

        sg1_ip = sg1.ip

        sg2_ip = sg2.ip

        sg1_url = sg1.url

        sg1_admin_url = sg1.admin.admin_url

        sg2_url = sg2.url

        sg2_admin_url = sg2.admin.admin_url

        if sg_ssl:

            protocol = "wss"

        sg1_blip_url = "{}://{}:4984/{}".format(protocol, sg1_ip, sg_db1)

        sg2_blip_url = "{}://{}:4984/{}".format(protocol, sg2_ip, sg_db2)

        auth = need_sgw_admin_auth and (RBAC_FULL_ADMIN['user'], RBAC_FULL_ADMIN['pwd']) or None

        sg_client.create_user(sg1_admin_url, sg_db1, name1, password="password", channels=channels1, auth=auth)

        sg_client.create_user(sg2_admin_url, sg_db2, name2, password="password", channels=channels2, auth=auth)

        # Create bulk doc json

        cbl_db1 = setup_customized_teardown_test["cbl_db1"]

        cbl_db2 = setup_customized_teardown_test["cbl_db2"]

        db.create_bulk_docs(num_of_docs, "Replication1", db=cbl_db1, channels=channels1)

        db.create_bulk_docs(num_of_docs, "Replication2", db=cbl_db2, channels=channels2)

        # 2. Do push replication to two SGS(each DB to each SG)

        replicator = Replication(base_url)

        cookie, session_id = sg_client.create_session(sg1_admin_url, sg_db1, name1, auth=auth)

        session1 = cookie, session_id

        authenticator = Authenticator(base_url)

        replicator_authenticator1 = authenticator.authentication(session_id, cookie, authentication_type="session")

        cookie, session_id = sg_client.create_session(sg2_admin_url, sg_db2, name2, auth=auth)

        session2 = cookie, session_id

        replicator_authenticator2 = authenticator.authentication(session_id, cookie, authentication_type="session")

        repl1 = replicator.configure_and_replicate(

            source_db=cbl_db1, replicator_authenticator=replicator_authenticator1, target_url=sg1_blip_url, replication_type="push")

        repl2 = replicator.configure_and_replicate(

            source_db=cbl_db2, replicator_authenticator=replicator_authenticator2, target_url=sg2_blip_url, replication_type="push")

        db.update_bulk_docs(cbl_db1, number_of_updates=18)

        db.update_bulk_docs(cbl_db2, number_of_updates=18)

        time.sleep(60)

        replicator.wait_until_replicator_idle(repl1)

        replicator.wait_until_replicator_idle(repl2)

        db.update_bulk_docs(cbl_db1, number_of_updates=17)

        db.update_bulk_docs(cbl_db2, number_of_updates=17)

        time.sleep(60)

        replicator.wait_until_replicator_idle(repl1)

        replicator.wait_until_replicator_idle(repl2)

        replicator.stop(repl1)

        replicator.stop(repl2)

>       time.sleep(280)

E       Failed: Timeout >1800s