Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-57898

[Provisioned/ColocatedServices]: GSI rebalance very slow under constant 80:20 workload of 5000 kv ops on 16vCPUs 64G instance(n2-standard-16)

    XMLWordPrintable

Details

    • Bug
    • Resolution: Cannot Reproduce
    • Critical
    • 7.6.0, 7.2.1
    • 7.2.1
    • secondary-index
    • Cloud: GCP
      Instance: n2-standard-16
      Services: kv:n1ql:gsi:fts:cbas collocated
      Image: couchbase-cloud-server-7-2-1-5855-v1-0-19

    Description

      1. Create a 3 node all services co-located cluster. Create a bucket, 2 collections, and load 100M items in each collection
      2. Create GSI/FTS indexes, CBAS datasets/indexes. Let ingestion for all to complete.
      3. Start 5000/s kv mutations with 80% reads and 20% sets
      4. Start the query load for all the services.
      5. Scale up cluster from 3->6
      6. Rebalance couldn't finish until the test failed due to rebalancing timeout of 5 hours and workload stopped. Later rebalance was successful when cluster was in idle state.

      System Indexes

      [
        {
          "indexes": {
            "bucket_id": "default0",
            "datastore_id": "http://127.0.0.1:8091",
            "id": "5c4b3e2b575a7fd7",
            "index_key": [
              "`country`",
              "`city`",
              "`price`",
              "`name`"
            ],
            "keyspace_id": "VolumeCollection1",
            "metadata": {
              "num_replica": 1
            },
            "name": "default0_idx_VolumeCollection1_1",
            "namespace_id": "default",
            "partition": "HASH(`country`,`city`)",
            "scope_id": "_default",
            "state": "online",
            "using": "gsi"
          }
        },
        {
          "indexes": {
            "bucket_id": "default0",
            "datastore_id": "http://127.0.0.1:8091",
            "id": "757df3e8c4cb5cb3",
            "index_key": [
              "`free_breakfast`",
              "`free_parking`",
              "`country`",
              "`city`"
            ],
            "keyspace_id": "VolumeCollection1",
            "metadata": {
              "num_replica": 1
            },
            "name": "default0_idx_VolumeCollection1_0",
            "namespace_id": "default",
            "partition": "HASH(`country`)",
            "scope_id": "_default",
            "state": "online",
            "using": "gsi"
          }
        },
        {
          "indexes": {
            "bucket_id": "default0",
            "datastore_id": "http://127.0.0.1:8091",
            "id": "b33d29015c271be3",
            "index_key": [
              "`country`",
              "(distinct (array ((`r`.`ratings`).`Check in / front desk`) for `r` in `reviews` end))",
              "array_count(`public_likes`)",
              "array_count(`reviews`) DESC",
              "`type`",
              "`phone`",
              "`price`",
              "`email`",
              "`address`",
              "`name`",
              "`url`"
            ],
            "keyspace_id": "VolumeCollection0",
            "metadata": {
              "num_replica": 1
            },
            "name": "default0_idx_VolumeCollection0_0",
            "namespace_id": "default",
            "partition": "HASH(`country`)",
            "scope_id": "_default",
            "state": "online",
            "using": "gsi"
          }
        },
        {
          "indexes": {
            "bucket_id": "default0",
            "datastore_id": "http://127.0.0.1:8091",
            "id": "a6421b4bd2c07eb3",
            "index_key": [
              "`free_breakfast`",
              "`type`",
              "`free_parking`",
              "array_count(`public_likes`)",
              "`price`",
              "`country`"
            ],
            "keyspace_id": "VolumeCollection0",
            "metadata": {
              "num_replica": 1
            },
            "name": "default0_idx_VolumeCollection0_1",
            "namespace_id": "default",
            "partition": "HASH(`type`)",
            "scope_id": "_default",
            "state": "online",
            "using": "gsi"
          }
        },
        {
          "indexes": {
            "bucket_id": "default0",
            "datastore_id": "http://127.0.0.1:8091",
            "id": "1c870b4d4fc17136",
            "index_key": [],
            "keyspace_id": "VolumeCollection0",
            "name": "default0VolumeCollection0_fts_idx_0",
            "namespace_id": "default",
            "scope_id": "_default",
            "state": "online",
            "using": "fts"
          }
        },
        {
          "indexes": {
            "bucket_id": "default0",
            "datastore_id": "http://127.0.0.1:8091",
            "id": "348dcf986da7f538",
            "index_key": [],
            "keyspace_id": "VolumeCollection0",
            "name": "default0VolumeCollection0_fts_idx_1",
            "namespace_id": "default",
            "scope_id": "_default",
            "state": "online",
            "using": "fts"
          }
        }
      ]
      

      Test

      sudo guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/couchbase_capella_volume_1_new.ini -p bucket_storage=magma,bucket_eviction_policy=fullEviction,rerun=False -t aGoodDoctor.hostedHospital.Murphy.test_rebalance,num_items=100000000,num_buckets=1,bucket_names=GleamBook,bucket_type=membase,iterations=5,batch_size=1000,sdk_timeout=60,log_level=debug,infra_log_level=debug,rerun=False,skip_cleanup=True,key_size=18,randomize_doc_size=False,randomize_value=True,maxttl=10,pc=20,gsi_nodes=3,cbas_nodes=3,fts_nodes=3,kv_nodes=3,n1ql_nodes=3,kv_disk=500,n1ql_disk=50,gsi_disk=500,fts_disk=500,cbas_disk=500,kv_compute=n2-standard-16,gsi_compute=n2-standard-16,n1ql_compute=n2-standard-16,fts_compute=n2-standard-16,cbas_compute=n2-standard-16,mutation_perc=20,key_type=CircularKey,capella_run=true,services=data:query:index:analytics:search,max_rebl_nodes=27,provider=GCP,region=us-east1,type=PD-SSD,size=500,skip_teardown_cleanup=false,wait_timeout=14400,index_timeout=28800,runtype=dedicated,sanity=True -m rest'
      

      cc: Deepkaran Salooja, Varun Velamuri

      Attachments

        Activity

          People

            ritesh.agarwal Ritesh Agarwal
            ritesh.agarwal Ritesh Agarwal
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              PagerDuty