Details
-
Bug
-
Resolution: Unresolved
-
Major
-
7.6.0
-
7.6.0-2076
-
Untriaged
-
-
0
-
Unknown
Description
Cluster config:
- 5 nodes collocated with data query search
- 4vCPU-16GB Ram
- Search -> 4gb per node -> 20GB Ram for FTS (very much overprovisioned)
I am seeing that with 10k docs (siftsmall Dataset) SEARCH() query is taking 1 min/query to compute, which is a bit disturbing as this has never been the case with non vector queries.
2024-02-02 06:40:17 | INFO | MainProcess | test_thread | [fts_vector_search.run_vector_query] ******************** Running Query # 40 - on index i1 ******************** |
2024-02-02 06:40:17 | INFO | MainProcess | test_thread | [fts_vector_search.run_vector_query] Running n1ql Query - SELECT COUNT(*) FROM `b1`.s1.c1 AS t1 WHERE SEARCH(t1, {'query': {'match_none': {}}, 'explain': True, 'fields': ['*'], 'knn': [{'field': 'vector_data', 'k': 100, 'vector': [49.0, 15.0, 18.0, 4.0, 0.0, 0.0, 4.0, 14.0, 97.0, 51.0, 95.0, 63.0, 0.0, 4.0, 17.0, 32.0, 104.0, 56.0, 78.0, 39.0, 0.0, 0.0, 3.0, 24.0, 29.0, 13.0, 22.0, 4.0, 0.0, 0.0, 0.0, 7.0, 114.0, 40.0, 28.0, 60.0, 5.0, 0.0, 0.0, 17.0, 114.0, 87.0, 97.0, 53.0, 0.0, 0.0, 0.0, 27.0, 47.0, 44.0, 91.0, 27.0, 16.0, 1.0, 0.0, 5.0, 5.0, 3.0, 10.0, 12.0, 5.0, 0.0, 0.0, 0.0, 69.0, 41.0, 14.0, 99.0, 50.0, 8.0, 33.0, 58.0, 109.0, 47.0, 3.0, 9.0, 3.0, 5.0, 90.0, 114.0, 10.0, 2.0, 2.0, 4.0, 19.0, 16.0, 103.0, 68.0, 0.0, 0.0, 3.0, 7.0, 8.0, 1.0, 5.0, 16.0, 13.0, 0.0, 1.0, 15.0, 24.0, 5.0, 20.0, 63.0, 44.0, 2.0, 2.0, 14.0, 28.0, 5.0, 78.0, 103.0, 32.0, 1.0, 2.0, 14.0, 51.0, 9.0, 89.0, 114.0, 15.0, 0.0, 0.0, 3.0, 5.0, 1.0, 15.0, 114.0]}]}); |
2024-02-02 06:40:17 | INFO | MainProcess | test_thread | [on_prem_rest_client.query_tool] query params : statement=SELECT+COUNT%28%2A%29+FROM+%60b1%60.s1.c1+AS+t1+WHERE+SEARCH%28t1%2C+%7B%27query%27%3A+%7B%27match_none%27%3A+%7B%7D%7D%2C+%27explain%27%3A+True%2C+%27fields%27%3A+%5B%27%2A%27%5D%2C+%27knn%27%3A+%5B%7B%27field%27%3A+%27vector_data%27%2C+%27k%27%3A+100%2C+%27vector%27%3A+%5B49.0%2C+15.0%2C+18.0%2C+4.0%2C+0.0%2C+0.0%2C+4.0%2C+14.0%2C+97.0%2C+51.0%2C+95.0%2C+63.0%2C+0.0%2C+4.0%2C+17.0%2C+32.0%2C+104.0%2C+56.0%2C+78.0%2C+39.0%2C+0.0%2C+0.0%2C+3.0%2C+24.0%2C+29.0%2C+13.0%2C+22.0%2C+4.0%2C+0.0%2C+0.0%2C+0.0%2C+7.0%2C+114.0%2C+40.0%2C+28.0%2C+60.0%2C+5.0%2C+0.0%2C+0.0%2C+17.0%2C+114.0%2C+87.0%2C+97.0%2C+53.0%2C+0.0%2C+0.0%2C+0.0%2C+27.0%2C+47.0%2C+44.0%2C+91.0%2C+27.0%2C+16.0%2C+1.0%2C+0.0%2C+5.0%2C+5.0%2C+3.0%2C+10.0%2C+12.0%2C+5.0%2C+0.0%2C+0.0%2C+0.0%2C+69.0%2C+41.0%2C+14.0%2C+99.0%2C+50.0%2C+8.0%2C+33.0%2C+58.0%2C+109.0%2C+47.0%2C+3.0%2C+9.0%2C+3.0%2C+5.0%2C+90.0%2C+114.0%2C+10.0%2C+2.0%2C+2.0%2C+4.0%2C+19.0%2C+16.0%2C+103.0%2C+68.0%2C+0.0%2C+0.0%2C+3.0%2C+7.0%2C+8.0%2C+1.0%2C+5.0%2C+16.0%2C+13.0%2C+0.0%2C+1.0%2C+15.0%2C+24.0%2C+5.0%2C+20.0%2C+63.0%2C+44.0%2C+2.0%2C+2.0%2C+14.0%2C+28.0%2C+5.0%2C+78.0%2C+103.0%2C+32.0%2C+1.0%2C+2.0%2C+14.0%2C+51.0%2C+9.0%2C+89.0%2C+114.0%2C+15.0%2C+0.0%2C+0.0%2C+3.0%2C+5.0%2C+1.0%2C+15.0%2C+114.0%5D%7D%5D%7D%29%3B |
2024-02-02 06:41:22 | INFO | MainProcess | test_thread | [fts_vector_search.run_vector_query] FTS Hits for N1QL query: 100 |
Query:
SELECT COUNT(*) FROM `b1`.s1.c1 AS t1 WHERE SEARCH(t1, {'query': {'match_none': {}}, 'explain': True, 'fields': ['*'], 'knn': [{'field': 'vector_data', 'k': 100, 'vector': [19.0, 28.0, 93.0, 19.0, 2.0, 3.0, 0.0, 0.0, 122.0, 105.0, 17.0, 0.0, 2.0, 95.0, 52.0, 4.0, 53.0, 15.0, 2.0, 0.0, 1.0, 96.0, 85.0, 95.0, 46.0, 1.0, 0.0, 0.0, 0.0, 0.0, 2.0, 43.0, 9.0, 13.0, 79.0, 11.0, 18.0, 126.0, 38.0, 22.0, 13.0, 13.0, 12.0, 0.0, 4.0, 126.0, 126.0, 23.0, 126.0, 2.0, 0.0, 0.0, 0.0, 41.0, 33.0, 105.0, 126.0, 4.0, 0.0, 0.0, 0.0, 3.0, 33.0, 51.0, 11.0, 0.0, 0.0, 0.0, 14.0, 126.0, 36.0, 30.0, 30.0, 6.0, 0.0, 0.0, 1.0, 25.0, 12.0, 25.0, 126.0, 28.0, 0.0, 0.0, 0.0, 3.0, 21.0, 49.0, 70.0, 4.0, 1.0, 0.0, 0.0, 35.0, 126.0, 42.0, 3.0, 0.0, 0.0, 2.0, 5.0, 4.0, 7.0, 4.0, 44.0, 10.0, 0.0, 0.0, 0.0, 0.0, 0.0, 13.0, 52.0, 44.0, 9.0, 0.0, 0.0, 1.0, 3.0, 7.0, 2.0, 13.0, 16.0, 0.0, 0.0, 5.0, 8.0, 1.0]}]}); |
Plan for query :
{
|
"#operator": "Authorize", |
"#planPreparedTime": "2024-02-02T16:20:03.915Z", |
"#stats": { |
"#phaseSwitches": 4, |
"execTime": "1.226µs", |
"servTime": "5.936µs" |
},
|
"privileges": { |
"List": [ |
{
|
"Priv": 7, |
"Props": 0, |
"Target": "default:b1.s1.c1" |
}
|
]
|
},
|
"~child": { |
"#operator": "Sequence", |
"#stats": { |
"#phaseSwitches": 2, |
"execTime": "3.171µs" |
},
|
"~children": [ |
{
|
"#operator": "IndexFtsSearch", |
"#stats": { |
"#heartbeatYields": 12, |
"#itemsIn": 100, |
"#itemsOut": 100, |
"#phaseSwitches": 403, |
"execTime": "770.084µs", |
"kernTime": "229.174µs", |
"servTime": "6.83925ms", |
"usedMemory": 25768 |
},
|
"as": "t1", |
"bucket": "b1", |
"covers": [ |
"cover (search(`t1`, {\"explain\": true, \"fields\": [\"*\"], \"knn\": [{\"field\": \"vector_data\", \"k\": 100, \"vector\": [19, 28, 93, 19, 2, 3, 0, 0, 122, 105, 17, 0, 2, 95, 52, 4, 53, 15, 2, 0, 1, 96, 85, 95, 46, 1, 0, 0, 0, 0, 2, 43, 9, 13, 79, 11, 18, 126, 38, 22, 13, 13, 12, 0, 4, 126, 126, 23, 126, 2, 0, 0, 0, 41, 33, 105, 126, 4, 0, 0, 0, 3, 33, 51, 11, 0, 0, 0, 14, 126, 36, 30, 30, 6, 0, 0, 1, 25, 12, 25, 126, 28, 0, 0, 0, 3, 21, 49, 70, 4, 1, 0, 0, 35, 126, 42, 3, 0, 0, 2, 5, 4, 7, 4, 44, 10, 0, 0, 0, 0, 0, 13, 52, 44, 9, 0, 0, 1, 3, 7, 2, 13, 16, 0, 0, 5, 8, 1]}], \"query\": {\"match_none\": {}}}))", |
"cover ((meta(`t1`).`id`))", |
"cover (search_score((`t1`.`out`)))", |
"cover (search_meta((`t1`.`out`)))" |
],
|
"index": "i2", |
"index_id": "59460ce51c661a07", |
"keyspace": "c1", |
"namespace": "default", |
"scope": "s1", |
"search_info": { |
"field": "\"\"", |
"outname": "out", |
"query": "{\"explain\": true, \"fields\": [\"*\"], \"knn\": [{\"field\": \"vector_data\", \"k\": 100, \"vector\": [19, 28, 93, 19, 2, 3, 0, 0, 122, 105, 17, 0, 2, 95, 52, 4, 53, 15, 2, 0, 1, 96, 85, 95, 46, 1, 0, 0, 0, 0, 2, 43, 9, 13, 79, 11, 18, 126, 38, 22, 13, 13, 12, 0, 4, 126, 126, 23, 126, 2, 0, 0, 0, 41, 33, 105, 126, 4, 0, 0, 0, 3, 33, 51, 11, 0, 0, 0, 14, 126, 36, 30, 30, 6, 0, 0, 1, 25, 12, 25, 126, 28, 0, 0, 0, 3, 21, 49, 70, 4, 1, 0, 0, 35, 126, 42, 3, 0, 0, 2, 5, 4, 7, 4, 44, 10, 0, 0, 0, 0, 0, 13, 52, 44, 9, 0, 0, 1, 3, 7, 2, 13, 16, 0, 0, 5, 8, 1]}], \"query\": {\"match_none\": {}}}" |
},
|
"using": "fts", |
"#time_normal": "00:00.007", |
"#time_absolute": 0.007609334 |
},
|
{
|
"#operator": "Sequence", |
"#stats": { |
"#phaseSwitches": 1, |
"execTime": "1.872µs", |
"usedMemory": 4272 |
},
|
"~children": [ |
{
|
"#operator": "Filter", |
"#stats": { |
"#itemsIn": 100, |
"#itemsOut": 100, |
"#phaseSwitches": 404, |
"execTime": "60.573µs", |
"kernTime": "7.802288ms", |
"usedMemory": 21476 |
},
|
"condition": "cover (search(`t1`, {\"explain\": true, \"fields\": [\"*\"], \"knn\": [{\"field\": \"vector_data\", \"k\": 100, \"vector\": [19, 28, 93, 19, 2, 3, 0, 0, 122, 105, 17, 0, 2, 95, 52, 4, 53, 15, 2, 0, 1, 96, 85, 95, 46, 1, 0, 0, 0, 0, 2, 43, 9, 13, 79, 11, 18, 126, 38, 22, 13, 13, 12, 0, 4, 126, 126, 23, 126, 2, 0, 0, 0, 41, 33, 105, 126, 4, 0, 0, 0, 3, 33, 51, 11, 0, 0, 0, 14, 126, 36, 30, 30, 6, 0, 0, 1, 25, 12, 25, 126, 28, 0, 0, 0, 3, 21, 49, 70, 4, 1, 0, 0, 35, 126, 42, 3, 0, 0, 2, 5, 4, 7, 4, 44, 10, 0, 0, 0, 0, 0, 13, 52, 44, 9, 0, 0, 1, 3, 7, 2, 13, 16, 0, 0, 5, 8, 1]}], \"query\": {\"match_none\": {}}}))", |
"#time_normal": "00:00.000", |
"#time_absolute": 0.000060573 |
},
|
{
|
"#operator": "InitialGroup", |
"#stats": { |
"#itemsIn": 100, |
"#itemsOut": 1, |
"#phaseSwitches": 206, |
"execTime": "217.541µs", |
"kernTime": "7.6691ms" |
},
|
"aggregates": [ |
"count(*)" |
],
|
"flags": 4, |
"group_keys": [], |
"#time_normal": "00:00.000", |
"#time_absolute": 0.000217541 |
}
|
],
|
"#time_normal": "00:00.000", |
"#time_absolute": 0.000001872 |
},
|
{
|
"#operator": "IntermediateGroup", |
"#stats": { |
"#itemsIn": 1, |
"#itemsOut": 1, |
"#phaseSwitches": 8, |
"execTime": "11.925µs", |
"kernTime": "7.890683ms", |
"usedMemory": 4272 |
},
|
"aggregates": [ |
"count(*)" |
],
|
"flags": 4, |
"group_keys": [], |
"#time_normal": "00:00.000", |
"#time_absolute": 0.000011925 |
},
|
{
|
"#operator": "FinalGroup", |
"#stats": { |
"#itemsIn": 1, |
"#itemsOut": 1, |
"#phaseSwitches": 8, |
"execTime": "14.385µs", |
"kernTime": "7.907847ms", |
"usedMemory": 4272 |
},
|
"aggregates": [ |
"count(*)" |
],
|
"flags": 4, |
"group_keys": [], |
"#time_normal": "00:00.000", |
"#time_absolute": 0.000014385 |
},
|
{
|
"#operator": "InitialProject", |
"#stats": { |
"#itemsIn": 1, |
"#itemsOut": 1, |
"#phaseSwitches": 7, |
"execTime": "31.89µs", |
"kernTime": "7.926845ms" |
},
|
"discard_original": true, |
"preserve_order": true, |
"result_terms": [ |
{
|
"expr": "count(*)" |
}
|
],
|
"#time_normal": "00:00.000", |
"#time_absolute": 0.00003189 |
},
|
{
|
"#operator": "Stream", |
"#stats": { |
"#itemsIn": 1, |
"#itemsOut": 1, |
"#phaseSwitches": 2, |
"execTime": "19.293µs" |
},
|
"serializable": true, |
"#time_normal": "00:00.000", |
"#time_absolute": 0.000019293 |
}
|
],
|
"#time_normal": "00:00.000", |
"#time_absolute": 0.000003171 |
},
|
"~versions": [ |
"7.6.0-N1QL", |
"7.6.0-2076-enterprise" |
],
|
"#time_normal": "00:00.000", |
"#time_absolute": 0.000007161999999999999 |
}
|
Plan with count(meta().id) instead of count( *) - No Change
{
|
"#operator": "Authorize", |
"#planPreparedTime": "2024-02-02T16:25:49.447Z", |
"#stats": { |
"#phaseSwitches": 4, |
"execTime": "1.792µs", |
"servTime": "385.98µs" |
},
|
"privileges": { |
"List": [ |
{
|
"Priv": 7, |
"Props": 0, |
"Target": "default:b1.s1.c1" |
}
|
]
|
},
|
"~child": { |
"#operator": "Sequence", |
"#stats": { |
"#phaseSwitches": 2, |
"execTime": "5.655µs" |
},
|
"~children": [ |
{
|
"#operator": "IndexFtsSearch", |
"#stats": { |
"#heartbeatYields": 17, |
"#itemsIn": 100, |
"#itemsOut": 100, |
"#phaseSwitches": 403, |
"execTime": "772.601µs", |
"kernTime": "358.891µs", |
"servTime": "11.23327ms", |
"usedMemory": 25772 |
},
|
"as": "t1", |
"bucket": "b1", |
"covers": [ |
"cover (search(`t1`, {\"explain\": true, \"fields\": [\"*\"], \"knn\": [{\"field\": \"vector_data\", \"k\": 100, \"vector\": [19, 28, 93, 19, 2, 3, 0, 0, 122, 105, 17, 0, 2, 95, 52, 4, 53, 15, 2, 0, 1, 96, 85, 95, 46, 1, 0, 0, 0, 0, 2, 43, 9, 13, 79, 11, 18, 126, 38, 22, 13, 13, 12, 0, 4, 126, 126, 23, 126, 2, 0, 0, 0, 41, 33, 105, 126, 4, 0, 0, 0, 3, 33, 51, 11, 0, 0, 0, 14, 126, 36, 30, 30, 6, 0, 0, 1, 25, 12, 25, 126, 28, 0, 0, 0, 3, 21, 49, 70, 4, 1, 0, 0, 35, 126, 42, 3, 0, 0, 2, 5, 4, 7, 4, 44, 10, 0, 0, 0, 0, 0, 13, 52, 44, 9, 0, 0, 1, 3, 7, 2, 13, 16, 0, 0, 5, 8, 1]}], \"query\": {\"match_none\": {}}}))", |
"cover ((meta(`t1`).`id`))", |
"cover (search_score((`t1`.`out`)))", |
"cover (search_meta((`t1`.`out`)))" |
],
|
"index": "i2", |
"index_id": "59460ce51c661a07", |
"keyspace": "c1", |
"namespace": "default", |
"scope": "s1", |
"search_info": { |
"field": "\"\"", |
"outname": "out", |
"query": "{\"explain\": true, \"fields\": [\"*\"], \"knn\": [{\"field\": \"vector_data\", \"k\": 100, \"vector\": [19, 28, 93, 19, 2, 3, 0, 0, 122, 105, 17, 0, 2, 95, 52, 4, 53, 15, 2, 0, 1, 96, 85, 95, 46, 1, 0, 0, 0, 0, 2, 43, 9, 13, 79, 11, 18, 126, 38, 22, 13, 13, 12, 0, 4, 126, 126, 23, 126, 2, 0, 0, 0, 41, 33, 105, 126, 4, 0, 0, 0, 3, 33, 51, 11, 0, 0, 0, 14, 126, 36, 30, 30, 6, 0, 0, 1, 25, 12, 25, 126, 28, 0, 0, 0, 3, 21, 49, 70, 4, 1, 0, 0, 35, 126, 42, 3, 0, 0, 2, 5, 4, 7, 4, 44, 10, 0, 0, 0, 0, 0, 13, 52, 44, 9, 0, 0, 1, 3, 7, 2, 13, 16, 0, 0, 5, 8, 1]}], \"query\": {\"match_none\": {}}}" |
},
|
"using": "fts", |
"#time_normal": "00:00.012", |
"#time_absolute": 0.012005871 |
},
|
{
|
"#operator": "Sequence", |
"#stats": { |
"#phaseSwitches": 1, |
"execTime": "3.084µs", |
"usedMemory": 4268 |
},
|
"~children": [ |
{
|
"#operator": "Filter", |
"#stats": { |
"#heartbeatYields": 2, |
"#itemsIn": 100, |
"#itemsOut": 100, |
"#phaseSwitches": 404, |
"execTime": "63.513µs", |
"kernTime": "12.335808ms", |
"usedMemory": 25768 |
},
|
"condition": "cover (search(`t1`, {\"explain\": true, \"fields\": [\"*\"], \"knn\": [{\"field\": \"vector_data\", \"k\": 100, \"vector\": [19, 28, 93, 19, 2, 3, 0, 0, 122, 105, 17, 0, 2, 95, 52, 4, 53, 15, 2, 0, 1, 96, 85, 95, 46, 1, 0, 0, 0, 0, 2, 43, 9, 13, 79, 11, 18, 126, 38, 22, 13, 13, 12, 0, 4, 126, 126, 23, 126, 2, 0, 0, 0, 41, 33, 105, 126, 4, 0, 0, 0, 3, 33, 51, 11, 0, 0, 0, 14, 126, 36, 30, 30, 6, 0, 0, 1, 25, 12, 25, 126, 28, 0, 0, 0, 3, 21, 49, 70, 4, 1, 0, 0, 35, 126, 42, 3, 0, 0, 2, 5, 4, 7, 4, 44, 10, 0, 0, 0, 0, 0, 13, 52, 44, 9, 0, 0, 1, 3, 7, 2, 13, 16, 0, 0, 5, 8, 1]}], \"query\": {\"match_none\": {}}}))", |
"#time_normal": "00:00.000", |
"#time_absolute": 0.000063513 |
},
|
{
|
"#operator": "InitialGroup", |
"#stats": { |
"#itemsIn": 100, |
"#itemsOut": 1, |
"#phaseSwitches": 206, |
"execTime": "312.807µs", |
"kernTime": "12.133627ms" |
},
|
"aggregates": [ |
"count(cover ((meta(`t1`).`id`)))" |
],
|
"flags": 4, |
"group_keys": [], |
"#time_normal": "00:00.000", |
"#time_absolute": 0.000312807 |
}
|
],
|
"#time_normal": "00:00.000", |
"#time_absolute": 0.000003084 |
},
|
{
|
"#operator": "IntermediateGroup", |
"#stats": { |
"#itemsIn": 1, |
"#itemsOut": 1, |
"#phaseSwitches": 8, |
"execTime": "22.186µs", |
"kernTime": "12.452265ms", |
"usedMemory": 4268 |
},
|
"aggregates": [ |
"count(cover ((meta(`t1`).`id`)))" |
],
|
"flags": 4, |
"group_keys": [], |
"#time_normal": "00:00.000", |
"#time_absolute": 0.000022186 |
},
|
{
|
"#operator": "FinalGroup", |
"#stats": { |
"#itemsIn": 1, |
"#itemsOut": 1, |
"#phaseSwitches": 8, |
"execTime": "21.396µs", |
"kernTime": "12.483196ms", |
"usedMemory": 4268 |
},
|
"aggregates": [ |
"count(cover ((meta(`t1`).`id`)))" |
],
|
"flags": 4, |
"group_keys": [], |
"#time_normal": "00:00.000", |
"#time_absolute": 0.000021396 |
},
|
{
|
"#operator": "InitialProject", |
"#stats": { |
"#itemsIn": 1, |
"#itemsOut": 1, |
"#phaseSwitches": 7, |
"execTime": "58.78µs", |
"kernTime": "12.509891ms" |
},
|
"discard_original": true, |
"preserve_order": true, |
"result_terms": [ |
{
|
"expr": "count(cover ((meta(`t1`).`id`)))" |
}
|
],
|
"#time_normal": "00:00.000", |
"#time_absolute": 0.00005878 |
},
|
{
|
"#operator": "Stream", |
"#stats": { |
"#itemsIn": 1, |
"#itemsOut": 1, |
"#phaseSwitches": 2, |
"execTime": "9.295µs" |
},
|
"serializable": true, |
"#time_normal": "00:00.000", |
"#time_absolute": 0.000009295 |
}
|
],
|
"#time_normal": "00:00.000", |
"#time_absolute": 0.000005655 |
},
|
"~versions": [ |
"7.6.0-N1QL", |
"7.6.0-2076-enterprise" |
],
|
"#time_normal": "00:00.000", |
"#time_absolute": 0.00038777200000000004 |
}
|
Toggling explain -> true /False also don't have any effect with the current behaviour.
Running on capella backend server directly also has the same latency.
