Details
-
Bug
-
Resolution: Cannot Reproduce
-
Critical
-
7.2.1
-
Cloud: GCP
Instance: n2-standard-16
Services: kv:n1ql:gsi:fts:cbas collocated
Image: couchbase-cloud-server-7-2-1-5855-v1-0-19
-
Untriaged
-
0
-
Unknown
Description
- Create a 3 node all services co-located cluster. Create a bucket, 2 collections, and load 100M items in each collection
- Create GSI/FTS indexes, CBAS datasets/indexes. Let ingestion for all to complete.
- Start 5000/s kv mutations with 80% reads and 20% sets
- Start the query load for all the services.
- Scale up cluster from 3->6
- Rebalance couldn't finish until the test failed due to rebalancing timeout of 5 hours and workload stopped. Later rebalance was successful when cluster was in idle state.
|
System Indexes |
[
|
{
|
"indexes": {
|
"bucket_id": "default0",
|
"datastore_id": "http://127.0.0.1:8091",
|
"id": "5c4b3e2b575a7fd7",
|
"index_key": [
|
"`country`",
|
"`city`",
|
"`price`",
|
"`name`"
|
],
|
"keyspace_id": "VolumeCollection1",
|
"metadata": {
|
"num_replica": 1
|
},
|
"name": "default0_idx_VolumeCollection1_1",
|
"namespace_id": "default",
|
"partition": "HASH(`country`,`city`)",
|
"scope_id": "_default",
|
"state": "online",
|
"using": "gsi"
|
}
|
},
|
{
|
"indexes": {
|
"bucket_id": "default0",
|
"datastore_id": "http://127.0.0.1:8091",
|
"id": "757df3e8c4cb5cb3",
|
"index_key": [
|
"`free_breakfast`",
|
"`free_parking`",
|
"`country`",
|
"`city`"
|
],
|
"keyspace_id": "VolumeCollection1",
|
"metadata": {
|
"num_replica": 1
|
},
|
"name": "default0_idx_VolumeCollection1_0",
|
"namespace_id": "default",
|
"partition": "HASH(`country`)",
|
"scope_id": "_default",
|
"state": "online",
|
"using": "gsi"
|
}
|
},
|
{
|
"indexes": {
|
"bucket_id": "default0",
|
"datastore_id": "http://127.0.0.1:8091",
|
"id": "b33d29015c271be3",
|
"index_key": [
|
"`country`",
|
"(distinct (array ((`r`.`ratings`).`Check in / front desk`) for `r` in `reviews` end))",
|
"array_count(`public_likes`)",
|
"array_count(`reviews`) DESC",
|
"`type`",
|
"`phone`",
|
"`price`",
|
"`email`",
|
"`address`",
|
"`name`",
|
"`url`"
|
],
|
"keyspace_id": "VolumeCollection0",
|
"metadata": {
|
"num_replica": 1
|
},
|
"name": "default0_idx_VolumeCollection0_0",
|
"namespace_id": "default",
|
"partition": "HASH(`country`)",
|
"scope_id": "_default",
|
"state": "online",
|
"using": "gsi"
|
}
|
},
|
{
|
"indexes": {
|
"bucket_id": "default0",
|
"datastore_id": "http://127.0.0.1:8091",
|
"id": "a6421b4bd2c07eb3",
|
"index_key": [
|
"`free_breakfast`",
|
"`type`",
|
"`free_parking`",
|
"array_count(`public_likes`)",
|
"`price`",
|
"`country`"
|
],
|
"keyspace_id": "VolumeCollection0",
|
"metadata": {
|
"num_replica": 1
|
},
|
"name": "default0_idx_VolumeCollection0_1",
|
"namespace_id": "default",
|
"partition": "HASH(`type`)",
|
"scope_id": "_default",
|
"state": "online",
|
"using": "gsi"
|
}
|
},
|
{
|
"indexes": {
|
"bucket_id": "default0",
|
"datastore_id": "http://127.0.0.1:8091",
|
"id": "1c870b4d4fc17136",
|
"index_key": [],
|
"keyspace_id": "VolumeCollection0",
|
"name": "default0VolumeCollection0_fts_idx_0",
|
"namespace_id": "default",
|
"scope_id": "_default",
|
"state": "online",
|
"using": "fts"
|
}
|
},
|
{
|
"indexes": {
|
"bucket_id": "default0",
|
"datastore_id": "http://127.0.0.1:8091",
|
"id": "348dcf986da7f538",
|
"index_key": [],
|
"keyspace_id": "VolumeCollection0",
|
"name": "default0VolumeCollection0_fts_idx_1",
|
"namespace_id": "default",
|
"scope_id": "_default",
|
"state": "online",
|
"using": "fts"
|
}
|
}
|
]
|
|
Test |
sudo guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/couchbase_capella_volume_1_new.ini -p bucket_storage=magma,bucket_eviction_policy=fullEviction,rerun=False -t aGoodDoctor.hostedHospital.Murphy.test_rebalance,num_items=100000000,num_buckets=1,bucket_names=GleamBook,bucket_type=membase,iterations=5,batch_size=1000,sdk_timeout=60,log_level=debug,infra_log_level=debug,rerun=False,skip_cleanup=True,key_size=18,randomize_doc_size=False,randomize_value=True,maxttl=10,pc=20,gsi_nodes=3,cbas_nodes=3,fts_nodes=3,kv_nodes=3,n1ql_nodes=3,kv_disk=500,n1ql_disk=50,gsi_disk=500,fts_disk=500,cbas_disk=500,kv_compute=n2-standard-16,gsi_compute=n2-standard-16,n1ql_compute=n2-standard-16,fts_compute=n2-standard-16,cbas_compute=n2-standard-16,mutation_perc=20,key_type=CircularKey,capella_run=true,services=data:query:index:analytics:search,max_rebl_nodes=27,provider=GCP,region=us-east1,type=PD-SSD,size=500,skip_teardown_cleanup=false,wait_timeout=14400,index_timeout=28800,runtype=dedicated,sanity=True -m rest'
|