Details
-
Bug
-
Resolution: Not a Bug
-
Critical
-
None
-
7.2.1
-
AWS
Enterprise Edition 7.2.1 build 5921
-
Untriaged
-
-
0
-
Unknown
Description
|
Cluster Config |
[
|
{
|
"compute": "c5.12xlarge",
|
"services":
|
[
|
"data"
|
],
|
"size": 3,
|
"storage":
|
{
|
"IOPS": 16000,
|
"size": 10250,
|
"type": "GP3"
|
}
|
},
|
{
|
"compute": "m5.16xlarge",
|
"services":
|
[
|
"index"
|
],
|
"size": 3,
|
"storage":
|
{
|
"IOPS": 16000,
|
"size": 10250,
|
"type": "GP3"
|
}
|
},
|
{
|
"compute": "c5.9xlarge",
|
"services":
|
[
|
"query"
|
],
|
"size": 2,
|
"storage":
|
{
|
"IOPS": 3000,
|
"size": 50,
|
"type": "GP3"
|
}
|
}
|
]
|
- Load 10B items in 2 collections. 5B in each.
- Build GSI indexes. 2 on 1 collection and 1 on another.
- Start a n1ql load asynchronously.
- Trigger compute scale UP for KV, GSI
- Query doing KV fetches is taking extremely long
|
Last 100 Query execution Data |
SELECT phaseTimes.`fetch` AS kv_fetch_time,
|
phaseTimes.indexScan AS gsi_scan_time,
|
phaseCounts.`fetch` AS kv_fetch_count,
|
phaseCounts.`indexScan` AS gsi_scan_count
|
FROM system:completed_requests
|
LIMIT 5;
|
|
|
[
|
{
|
"gsi_scan_count": 5075,
|
"gsi_scan_time": "12.418986ms",
|
"kv_fetch_count": 5075,
|
"kv_fetch_time": "39.364114675s"
|
},
|
{
|
"gsi_scan_count": 5016,
|
"gsi_scan_time": "21.174313ms",
|
"kv_fetch_count": 5016,
|
"kv_fetch_time": "37.562094079s"
|
},
|
{
|
"gsi_scan_count": 5062,
|
"gsi_scan_time": "13.506264ms",
|
"kv_fetch_count": 5062,
|
"kv_fetch_time": "36.206523299s"
|
},
|
{
|
"gsi_scan_count": 5075,
|
"gsi_scan_time": "13.039354ms",
|
"kv_fetch_count": 5075,
|
"kv_fetch_time": "33.925197017s"
|
},
|
{
|
"gsi_scan_count": 4978,
|
"gsi_scan_time": "16.455117ms",
|
"kv_fetch_count": 4978,
|
"kv_fetch_time": "30.395646805s"
|
},
|
{
|
"gsi_scan_count": 5015,
|
"gsi_scan_time": "13.156055ms",
|
"kv_fetch_count": 5015,
|
"kv_fetch_time": "38.512460594s"
|
},
|
{
|
"gsi_scan_count": 5012,
|
"gsi_scan_time": "16.066854ms",
|
"kv_fetch_count": 5012,
|
"kv_fetch_time": "33.165496156s"
|
},
|
{
|
"gsi_scan_count": 5019,
|
"gsi_scan_time": "15.920364ms",
|
"kv_fetch_count": 5019,
|
"kv_fetch_time": "41.280662071s"
|
},
|
{
|
"gsi_scan_count": 5131,
|
"gsi_scan_time": "13.484352ms",
|
"kv_fetch_count": 5131,
|
"kv_fetch_time": "37.198757564s"
|
},
|
{
|
"gsi_scan_count": 4924,
|
"gsi_scan_time": "16.842441ms",
|
"kv_fetch_count": 4924,
|
"kv_fetch_time": "32.760752587s"
|
},
|
{
|
"gsi_scan_count": 5052,
|
"gsi_scan_time": "14.157209ms",
|
"kv_fetch_count": 5052,
|
"kv_fetch_time": "34.116433675s"
|
},
|
{
|
"gsi_scan_count": 5005,
|
"gsi_scan_time": "15.944356ms",
|
"kv_fetch_count": 5005,
|
"kv_fetch_time": "33.636370703s"
|
},
|
{
|
"gsi_scan_count": 4753,
|
"gsi_scan_time": "22.183125ms",
|
"kv_fetch_count": 4753,
|
"kv_fetch_time": "33.375123457s"
|
},
|
{
|
"gsi_scan_count": 4910,
|
"gsi_scan_time": "12.412672ms",
|
"kv_fetch_count": 4910,
|
"kv_fetch_time": "32.465585877s"
|
},
|
{
|
"gsi_scan_count": 4989,
|
"gsi_scan_time": "14.534644ms",
|
"kv_fetch_count": 4989,
|
"kv_fetch_time": "33.339534529s"
|
},
|
{
|
"gsi_scan_count": 5061,
|
"gsi_scan_time": "17.162094ms",
|
"kv_fetch_count": 5061,
|
"kv_fetch_time": "38.849207174s"
|
},
|
{
|
"gsi_scan_count": 4921,
|
"gsi_scan_time": "14.336841ms",
|
"kv_fetch_count": 4921,
|
"kv_fetch_time": "36.812534311s"
|
},
|
{
|
"gsi_scan_count": 5065,
|
"gsi_scan_time": "19.214016ms",
|
"kv_fetch_count": 5065,
|
"kv_fetch_time": "34.198270678s"
|
},
|
{
|
"gsi_scan_count": 4912,
|
"gsi_scan_time": "14.248124ms",
|
"kv_fetch_count": 4912,
|
"kv_fetch_time": "31.033639549s"
|
},
|
{
|
"gsi_scan_count": 4968,
|
"gsi_scan_time": "12.766551ms",
|
"kv_fetch_count": 4968,
|
"kv_fetch_time": "30.41043462s"
|
},
|
{
|
"gsi_scan_count": 4961,
|
"gsi_scan_time": "12.372962ms",
|
"kv_fetch_count": 4961,
|
"kv_fetch_time": "32.268662491s"
|
},
|
{
|
"gsi_scan_count": 5015,
|
"gsi_scan_time": "9.319422ms",
|
"kv_fetch_count": 5015,
|
"kv_fetch_time": "32.282108671s"
|
},
|
{
|
"gsi_scan_count": 4980,
|
"gsi_scan_time": "15.33802ms",
|
"kv_fetch_count": 4980,
|
"kv_fetch_time": "32.446984985s"
|
},
|
{
|
"gsi_scan_count": 5072,
|
"gsi_scan_time": "15.761584ms",
|
"kv_fetch_count": 5072,
|
"kv_fetch_time": "32.999696351s"
|
},
|
{
|
"gsi_scan_count": 5014,
|
"gsi_scan_time": "15.856528ms",
|
"kv_fetch_count": 5014,
|
"kv_fetch_time": "32.952049527s"
|
},
|
{
|
"gsi_scan_count": 5086,
|
"gsi_scan_time": "7.467102ms",
|
"kv_fetch_count": 5086,
|
"kv_fetch_time": "31.035206741s"
|
},
|
{
|
"gsi_scan_count": 5210,
|
"gsi_scan_time": "14.222698ms",
|
"kv_fetch_count": 5210,
|
"kv_fetch_time": "37.585548646s"
|
},
|
{
|
"gsi_scan_count": 5059,
|
"gsi_scan_time": "12.687887ms",
|
"kv_fetch_count": 5059,
|
"kv_fetch_time": "35.516386803s"
|
},
|
{
|
"gsi_scan_count": 5052,
|
"gsi_scan_time": "6.648891ms",
|
"kv_fetch_count": 5052,
|
"kv_fetch_time": "38.594309441s"
|
},
|
{
|
"gsi_scan_count": 4956,
|
"gsi_scan_time": "13.612825ms",
|
"kv_fetch_count": 4956,
|
"kv_fetch_time": "31.528876534s"
|
},
|
{
|
"gsi_scan_count": 4945,
|
"gsi_scan_time": "15.27273ms",
|
"kv_fetch_count": 4945,
|
"kv_fetch_time": "35.874954023s"
|
},
|
{
|
"gsi_scan_count": 5050,
|
"gsi_scan_time": "13.647634ms",
|
"kv_fetch_count": 5050,
|
"kv_fetch_time": "32.622958674s"
|
},
|
{
|
"gsi_scan_count": 5083,
|
"gsi_scan_time": "14.39553ms",
|
"kv_fetch_count": 5083,
|
"kv_fetch_time": "35.406867842s"
|
},
|
{
|
"gsi_scan_count": 4950,
|
"gsi_scan_time": "16.174101ms",
|
"kv_fetch_count": 4950,
|
"kv_fetch_time": "36.281762539s"
|
},
|
{
|
"gsi_scan_count": 5004,
|
"gsi_scan_time": "13.096814ms",
|
"kv_fetch_count": 5004,
|
"kv_fetch_time": "40.243548712s"
|
},
|
{
|
"gsi_scan_count": 5116,
|
"gsi_scan_time": "14.736455ms",
|
"kv_fetch_count": 5116,
|
"kv_fetch_time": "39.683493701s"
|
},
|
{
|
"gsi_scan_count": 5038,
|
"gsi_scan_time": "9.956307ms",
|
"kv_fetch_count": 5038,
|
"kv_fetch_time": "34.620161077s"
|
},
|
{
|
"gsi_scan_count": 5032,
|
"gsi_scan_time": "14.369267ms",
|
"kv_fetch_count": 5032,
|
"kv_fetch_time": "36.841178778s"
|
},
|
{
|
"gsi_scan_count": 4998,
|
"gsi_scan_time": "13.417413ms",
|
"kv_fetch_count": 4998,
|
"kv_fetch_time": "37.471791111s"
|
},
|
{
|
"gsi_scan_count": 4980,
|
"gsi_scan_time": "14.308744ms",
|
"kv_fetch_count": 4980,
|
"kv_fetch_time": "36.226971442s"
|
},
|
{
|
"gsi_scan_count": 5044,
|
"gsi_scan_time": "17.426428ms",
|
"kv_fetch_count": 5044,
|
"kv_fetch_time": "35.525726017s"
|
},
|
{
|
"gsi_scan_count": 4934,
|
"gsi_scan_time": "16.567878ms",
|
"kv_fetch_count": 4934,
|
"kv_fetch_time": "35.285034339s"
|
},
|
{
|
"gsi_scan_count": 5032,
|
"gsi_scan_time": "14.438185ms",
|
"kv_fetch_count": 5032,
|
"kv_fetch_time": "35.949631866s"
|
},
|
{
|
"gsi_scan_count": 5029,
|
"gsi_scan_time": "20.30398ms",
|
"kv_fetch_count": 5029,
|
"kv_fetch_time": "33.808991967s"
|
},
|
{
|
"gsi_scan_count": 4972,
|
"gsi_scan_time": "16.044277ms",
|
"kv_fetch_count": 4972,
|
"kv_fetch_time": "36.234550861s"
|
},
|
{
|
"gsi_scan_count": 5025,
|
"gsi_scan_time": "14.860028ms",
|
"kv_fetch_count": 5025,
|
"kv_fetch_time": "36.17945219s"
|
},
|
{
|
"gsi_scan_count": 5060,
|
"gsi_scan_time": "13.790322ms",
|
"kv_fetch_count": 5060,
|
"kv_fetch_time": "35.582092716s"
|
},
|
{
|
"gsi_scan_count": 4985,
|
"gsi_scan_time": "17.267778ms",
|
"kv_fetch_count": 4985,
|
"kv_fetch_time": "32.444277587s"
|
},
|
{
|
"gsi_scan_count": 4999,
|
"gsi_scan_time": "14.290546ms",
|
"kv_fetch_count": 4999,
|
"kv_fetch_time": "36.159219288s"
|
},
|
{
|
"gsi_scan_count": 5025,
|
"gsi_scan_time": "19.130185ms",
|
"kv_fetch_count": 5025,
|
"kv_fetch_time": "34.119563918s"
|
},
|
{
|
"gsi_scan_count": 4980,
|
"gsi_scan_time": "11.477521ms",
|
"kv_fetch_count": 4980,
|
"kv_fetch_time": "32.658697474s"
|
},
|
{
|
"gsi_scan_count": 4958,
|
"gsi_scan_time": "7.527205ms",
|
"kv_fetch_count": 4958,
|
"kv_fetch_time": "33.471849228s"
|
},
|
{
|
"gsi_scan_count": 4786,
|
"gsi_scan_time": "8.078085ms",
|
"kv_fetch_count": 4786,
|
"kv_fetch_time": "34.471597389s"
|
},
|
{
|
"gsi_scan_count": 5103,
|
"gsi_scan_time": "16.898758ms",
|
"kv_fetch_count": 5103,
|
"kv_fetch_time": "35.192004218s"
|
},
|
{
|
"gsi_scan_count": 5046,
|
"gsi_scan_time": "17.169999ms",
|
"kv_fetch_count": 5046,
|
"kv_fetch_time": "36.883724813s"
|
},
|
{
|
"gsi_scan_count": 5071,
|
"gsi_scan_time": "12.777442ms",
|
"kv_fetch_count": 5071,
|
"kv_fetch_time": "35.726239038s"
|
},
|
{
|
"gsi_scan_count": 4953,
|
"gsi_scan_time": "16.801146ms",
|
"kv_fetch_count": 4953,
|
"kv_fetch_time": "26.857915358s"
|
},
|
{
|
"gsi_scan_count": 4897,
|
"gsi_scan_time": "13.596896ms",
|
"kv_fetch_count": 4897,
|
"kv_fetch_time": "34.038648746s"
|
},
|
{
|
"gsi_scan_count": 5100,
|
"gsi_scan_time": "14.33052ms",
|
"kv_fetch_count": 5100,
|
"kv_fetch_time": "35.257415945s"
|
},
|
{
|
"gsi_scan_count": 5006,
|
"gsi_scan_time": "14.048942ms",
|
"kv_fetch_count": 5006,
|
"kv_fetch_time": "33.956298582s"
|
},
|
{
|
"gsi_scan_count": 4968,
|
"gsi_scan_time": "19.165142ms",
|
"kv_fetch_count": 4968,
|
"kv_fetch_time": "29.392531261s"
|
},
|
{
|
"gsi_scan_count": 4965,
|
"gsi_scan_time": "11.555206ms",
|
"kv_fetch_count": 4965,
|
"kv_fetch_time": "35.000534645s"
|
},
|
{
|
"gsi_scan_count": 5048,
|
"gsi_scan_time": "14.006554ms",
|
"kv_fetch_count": 5048,
|
"kv_fetch_time": "29.693668278s"
|
},
|
{
|
"gsi_scan_count": 5028,
|
"gsi_scan_time": "14.953922ms",
|
"kv_fetch_count": 5028,
|
"kv_fetch_time": "32.021763833s"
|
},
|
{
|
"gsi_scan_count": 4900,
|
"gsi_scan_time": "16.981067ms",
|
"kv_fetch_count": 4900,
|
"kv_fetch_time": "32.446211587s"
|
},
|
{
|
"gsi_scan_count": 4861,
|
"gsi_scan_time": "13.343708ms",
|
"kv_fetch_count": 4861,
|
"kv_fetch_time": "31.456993818s"
|
},
|
{
|
"gsi_scan_count": 4955,
|
"gsi_scan_time": "8.845467ms",
|
"kv_fetch_count": 4955,
|
"kv_fetch_time": "35.400015454s"
|
},
|
{
|
"gsi_scan_count": 5011,
|
"gsi_scan_time": "7.745152ms",
|
"kv_fetch_count": 5011,
|
"kv_fetch_time": "38.32496889s"
|
},
|
{
|
"gsi_scan_count": 5121,
|
"gsi_scan_time": "14.425647ms",
|
"kv_fetch_count": 5121,
|
"kv_fetch_time": "40.246652795s"
|
},
|
{
|
"gsi_scan_count": 4955,
|
"gsi_scan_time": "13.687647ms",
|
"kv_fetch_count": 4955,
|
"kv_fetch_time": "33.46338143s"
|
},
|
{
|
"gsi_scan_count": 5154,
|
"gsi_scan_time": "14.562101ms",
|
"kv_fetch_count": 5154,
|
"kv_fetch_time": "37.837973638s"
|
},
|
{
|
"gsi_scan_count": 4907,
|
"gsi_scan_time": "13.765428ms",
|
"kv_fetch_count": 4907,
|
"kv_fetch_time": "34.604980115s"
|
},
|
{
|
"gsi_scan_count": 5041,
|
"gsi_scan_time": "8.819417ms",
|
"kv_fetch_count": 5041,
|
"kv_fetch_time": "36.37756311s"
|
},
|
{
|
"gsi_scan_count": 4981,
|
"gsi_scan_time": "14.544303ms",
|
"kv_fetch_count": 4981,
|
"kv_fetch_time": "38.736049593s"
|
},
|
{
|
"gsi_scan_count": 4983,
|
"gsi_scan_time": "14.038224ms",
|
"kv_fetch_count": 4983,
|
"kv_fetch_time": "32.454871731s"
|
},
|
{
|
"gsi_scan_count": 5003,
|
"gsi_scan_time": "15.90658ms",
|
"kv_fetch_count": 5003,
|
"kv_fetch_time": "34.690753285s"
|
},
|
{
|
"gsi_scan_count": 5075,
|
"gsi_scan_time": "13.909927ms",
|
"kv_fetch_count": 5075,
|
"kv_fetch_time": "35.318272321s"
|
},
|
{
|
"gsi_scan_count": 4994,
|
"gsi_scan_time": "10.814554ms",
|
"kv_fetch_count": 4994,
|
"kv_fetch_time": "34.263634858s"
|
},
|
{
|
"gsi_scan_count": 4984,
|
"gsi_scan_time": "15.37444ms",
|
"kv_fetch_count": 4984,
|
"kv_fetch_time": "30.90522284s"
|
},
|
{
|
"gsi_scan_count": 4933,
|
"gsi_scan_time": "13.277492ms",
|
"kv_fetch_count": 4933,
|
"kv_fetch_time": "30.214315771s"
|
},
|
{
|
"gsi_scan_count": 4992,
|
"gsi_scan_time": "9.491759ms",
|
"kv_fetch_count": 4992,
|
"kv_fetch_time": "31.753294832s"
|
},
|
{
|
"gsi_scan_count": 5027,
|
"gsi_scan_time": "11.827054ms",
|
"kv_fetch_count": 5027,
|
"kv_fetch_time": "34.683539833s"
|
},
|
{
|
"gsi_scan_count": 5015,
|
"gsi_scan_time": "12.833372ms",
|
"kv_fetch_count": 5015,
|
"kv_fetch_time": "30.107796674s"
|
},
|
{
|
"gsi_scan_count": 4974,
|
"gsi_scan_time": "13.942379ms",
|
"kv_fetch_count": 4974,
|
"kv_fetch_time": "31.658794381s"
|
},
|
{
|
"gsi_scan_count": 4983,
|
"gsi_scan_time": "13.875517ms",
|
"kv_fetch_count": 4983,
|
"kv_fetch_time": "34.22274362s"
|
},
|
{
|
"gsi_scan_count": 4969,
|
"gsi_scan_time": "16.418011ms",
|
"kv_fetch_count": 4969,
|
"kv_fetch_time": "29.696101445s"
|
},
|
{
|
"gsi_scan_count": 5024,
|
"gsi_scan_time": "15.650064ms",
|
"kv_fetch_count": 5024,
|
"kv_fetch_time": "35.54044192s"
|
},
|
{
|
"gsi_scan_count": 4874,
|
"gsi_scan_time": "15.491039ms",
|
"kv_fetch_count": 4874,
|
"kv_fetch_time": "31.623025526s"
|
},
|
{
|
"gsi_scan_count": 4902,
|
"gsi_scan_time": "13.655376ms",
|
"kv_fetch_count": 4902,
|
"kv_fetch_time": "37.140225241s"
|
},
|
{
|
"gsi_scan_count": 5002,
|
"gsi_scan_time": "12.856647ms",
|
"kv_fetch_count": 5002,
|
"kv_fetch_time": "33.851002966s"
|
},
|
{
|
"gsi_scan_count": 4954,
|
"gsi_scan_time": "25.731405ms",
|
"kv_fetch_count": 4954,
|
"kv_fetch_time": "30.160074629s"
|
},
|
{
|
"gsi_scan_count": 5127,
|
"gsi_scan_time": "17.42957ms",
|
"kv_fetch_count": 5127,
|
"kv_fetch_time": "30.848364005s"
|
},
|
{
|
"gsi_scan_count": 4963,
|
"gsi_scan_time": "12.997153ms",
|
"kv_fetch_count": 4963,
|
"kv_fetch_time": "33.211340288s"
|
},
|
{
|
"gsi_scan_count": 5140,
|
"gsi_scan_time": "16.341619ms",
|
"kv_fetch_count": 5140,
|
"kv_fetch_time": "34.065975322s"
|
},
|
{
|
"gsi_scan_count": 5039,
|
"gsi_scan_time": "15.363198ms",
|
"kv_fetch_count": 5039,
|
"kv_fetch_time": "30.598933426s"
|
},
|
{
|
"gsi_scan_count": 4982,
|
"gsi_scan_time": "13.331754ms",
|
"kv_fetch_count": 4982,
|
"kv_fetch_time": "33.000875257s"
|
},
|
{
|
"gsi_scan_count": 5029,
|
"gsi_scan_time": "16.728622ms",
|
"kv_fetch_count": 5029,
|
"kv_fetch_time": "38.677927162s"
|
},
|
{
|
"gsi_scan_count": 4933,
|
"gsi_scan_time": "14.575137ms",
|
"kv_fetch_count": 4933,
|
"kv_fetch_time": "32.665131501s"
|
},
|
{
|
"gsi_scan_count": 5024,
|
"gsi_scan_time": "15.00209ms",
|
"kv_fetch_count": 5024,
|
"kv_fetch_time": "36.908212764s"
|
},
|
{
|
"gsi_scan_count": 5043,
|
"gsi_scan_time": "13.554811ms",
|
"kv_fetch_count": 5043,
|
"kv_fetch_time": "30.831414887s"
|
}
|
]
|
Query Types:
Q0 == SELECT meta().id, conversationId, lastMessageDate, lastReadDate, unreadCount FROM {} WHERE uid = $uid ORDER BY lastMessageDate DESC LIMIT $N;
|
|
|
Q1 == SELECT uid, conversationId FROM {} WHERE conversationId IN [$conversationId1, $conversationId2]
|
|
|
Q2 == SELECT meta().id AS _id, uid, type, content, url, timestamp, width, height, clickable, roomId, roomTitle, roomStreamers, actions, pixel FROM {} WHERE conversationId = $conversationId ORDER BY timestamp DESC LIMIT $N
|
|
|
Q3 == SELECT COUNT(*) AS nb FROM {} WHERE conversationId = $conversationId
|
N1QL Query Statistics
+----------+---------------+----------------+-----------------+------------------+-------------------+-----------------+-----------------+
|
| Bucket | Total Queries | Failed Queries | Success Queries | Rejected Queries | Cancelled Queries | Timeout Queries | Errored Queries |
|
+----------+---------------+----------------+-----------------+------------------+-------------------+-----------------+-----------------+
|
| default0 | count(92289) | count(0) | count(72642) | count(0) | count(0) | count(19263) | count(0) |
|
+----------+---------------+----------------+-----------------+------------------+-------------------+-----------------+-----------------+
|
|
|
+----------+-------+-------+------------------------+
|
| Bucket | Query | Count | Avg Execution Time(ms) |
|
+----------+-------+-------+------------------------+
|
| default0 | Q3 | 18595 | 28.8128802945 |
|
| default0 | Q2 | 16660 | 32643.1910235 |
|
| default0 | Q0 | 18645 | 2.19920518353 |
|
| default0 | Q1 | 18760 | 52.0205795256 |
|
+----------+-------+-------+------------------------+
|
