All tests on a 24 vCPU system 2.2GHz single node with 64gb of system RAM 
and 45000Mib of Search Quota and a jemalloc bash wrapper to launch the 
renamed cbft binary of cbft.7.6.1.20240327

#!/bin/bash
export LD_PRELOAD=/usr/local/lib/libjemalloc.so
export MALLOC_CONF="background_thread:true,dirty_decay_ms:20000,muzzy_decay_ms:20000"
/space/linuxbrew/couch/install/bin/cbft.7.6.1.20240327 "$@"

We see that every concurrent client needs an additoinal 398MiB on average. Thus
too many requests can cause system freezes and lead to OOMs.

###########################################################################

cb_fts_bench/cb_fts_bench -m POST -H "Content-Type: application/json" \
    -t 180s -d 5m -c 6 -f ./vindex_query_120sec_ctl.json \
    -u ${CB_USERNAME}:${CB_PASSWORD} \
    http://$CB_HOSTNAME:8094/api/bucket/target/scope/_default/index/vindex/query

RES during 5minute test
 2.1g  * idle
14.7g  6 clients


Statistics        Avg      Stdev        Max
  Reqs/sec         8.53      19.58     197.42
  Latency      689.57ms    87.69ms      1.06s
  HTTP codes:
    1xx - 0, 2xx - 2612, 3xx - 0, 4xx - 0, 5xx - 0
    others - 0
  Throughput:   301.61KB/s
  HTTP 429 retries            0, pct.     0.000% across 2612 reqs
  Ave Sizes
    resp.body:             518 bytes (across      2612 reqs)
    resp.body.hit[]:       375 bytes (across      2612 reqs having hits [100.00%])
  Other
    total_hits                7836, ave     3.000
    tot_hits_docreads         7836, ave     3.000
    status.total              2612
    status.failed                0
    status.successful         2612
    Test Time in Sec.      300.000

(about 1m 30s decay to RES=2.1g)
2841004 linuxbr+  20   0  135.7g   4.8g   1.0g S  15.8   7.7   8433:30 cbft.7.6.1.2024
2841004 linuxbr+  20   0  135.7g   2.4g   1.0g S   1.7   3.8   8433:30 cbft.7.6.1.2024
2841004 linuxbr+  20   0  135.7g   2.3g   1.0g S   2.0   3.7   8433:31 cbft.7.6.1.2024
2841004 linuxbr+  20   0  135.7g   2.1g   1.0g S   2.6   3.3   8433:33 cbft.7.6.1.2024

###########################################################################

cb_fts_bench/cb_fts_bench -m POST -H "Content-Type: application/json" \
    -t 180s -d 5m -c 16 -f ./vindex_query_120sec_ctl.json \
    -u ${CB_USERNAME}:${CB_PASSWORD} \
    http://$CB_HOSTNAME:8094/api/bucket/target/scope/_default/index/vindex/query

RES during 5minute test
 2.1g  * idle
19.2g 16 clients  [add 10 clients 19.2 - 14.7]
                  that's 4.5g for 10 clients or 450MiB/client

Statistics        Avg      Stdev        Max
  Reqs/sec         8.36      19.12     223.54
  Latency         1.87s   269.83ms      2.98s
  HTTP codes:
    1xx - 0, 2xx - 2571, 3xx - 0, 4xx - 0, 5xx - 0
    others - 0
  Throughput:   296.50KB/s
  HTTP 429 retries            0, pct.     0.000% across 2571 reqs
  Ave Sizes
    resp.body:             518 bytes (across      2571 reqs)
    resp.body.hit[]:       375 bytes (across      2571 reqs having hits [100.00%])
  Other
    total_hits                7713, ave     3.000
    tot_hits_docreads         7713, ave     3.000
    status.total              2571
    status.failed                0
    status.successful         2571
    Test Time in Sec.      300.000

(about 1m 30s decay to RES=2.1g)
2841004 linuxbr+  20   0  137.6g   8.1g   1.0g S  26.5  12.9   8548:27 cbft.7.6.1.2024
2841004 linuxbr+  20   0  137.6g   2.8g   1.0g S   6.6   4.5   8548:28 cbft.7.6.1.2024
2841004 linuxbr+  20   0  137.6g   2.5g   1.0g S   1.7   4.1   8548:30 cbft.7.6.1.2024
2841004 linuxbr+  20   0  137.6g   2.2g   1.0g S  10.2   3.6   8548:32 cbft.7.6.1.2024
2841004 linuxbr+  20   0  137.6g   2.1g   1.0g S   9.6   3.3   8548:35 cbft.7.6.1.2024


###########################################################################

cb_fts_bench/cb_fts_bench -m POST -H "Content-Type: application/json" \
    -t 180s -d 5m -c 36 -f ./vindex_query_120sec_ctl.json \
    -u ${CB_USERNAME}:${CB_PASSWORD} \
    http://$CB_HOSTNAME:8094/api/bucket/target/scope/_default/index/vindex/query


RES during 5minute test
 2.1g  * idle
29.5g 36 clients  [add 10 clients 29.5 - 19.2]
                  that's 10.3g for 20 clients or 515MiB/client

Statistics        Avg      Stdev        Max
  Reqs/sec         8.27      19.45     207.25
  Latency         4.26s   508.62ms      6.32s
  HTTP codes:
    1xx - 0, 2xx - 2544, 3xx - 0, 4xx - 0, 5xx - 0
    others - 0
  Throughput:   292.90KB/s
  HTTP 429 retries            0, pct.     0.000% across 2544 reqs
  Ave Sizes
    resp.body:             519 bytes (across      2544 reqs)
    resp.body.hit[]:       375 bytes (across      2544 reqs having hits [100.00%])
  Other
    total_hits                7632, ave     3.000
    tot_hits_docreads         7632, ave     3.000
    status.total              2544
    status.failed                0
    status.successful         2544
    Test Time in Sec.      301.000


2841004 linuxbr+  20   0  141.5g   2.4g   1.0g S   3.3   3.9   8664:13 cbft.7.6.1.2024


###########################################################################

cb_fts_bench/cb_fts_bench -m POST -H "Content-Type: application/json" \
    -t 180s -d 5m -c 56 -f ./vindex_query_120sec_ctl.json \
    -u ${CB_USERNAME}:${CB_PASSWORD} \
    http://$CB_HOSTNAME:8094/api/bucket/target/scope/_default/index/vindex/query


RES during 5minute test
 2.1g  * idle
37.1g 56 clients  [add 20 clients  37.1 - 29.5]
                  that's 7.6g for 20 clients or 380MiB/client
		  [ add 50 clients 37.1 - 14.7
		  that's 22.4g for 20 clients or 448MiB/client

Statistics        Avg      Stdev        Max
  Reqs/sec         8.22      19.91     259.91
  Latency         6.67s      0.86s     10.38s
  HTTP codes:
    1xx - 0, 2xx - 2529, 3xx - 0, 4xx - 0, 5xx - 0
    others - 0
  Throughput:   290.65KB/s
  HTTP 429 retries            0, pct.     0.000% across 2529 reqs
  Ave Sizes
    resp.body:             519 bytes (across      2529 reqs)
    resp.body.hit[]:       375 bytes (across      2529 reqs having hits [100.00%])
  Other
    total_hits                7587, ave     3.000
    tot_hits_docreads         7587, ave     3.000
    status.total              2529
    status.failed                0
    status.successful         2529
    Test Time in Sec.      301.000

2841004 linuxbr+  20   0  146.8g  12.1g   1.0g S  54.8  19.3   8779:48 cbft.7.6.1.2024
2841004 linuxbr+  20   0  146.8g   5.7g   1.0g S  25.1   9.1   8779:50 cbft.7.6.1.2024
2841004 linuxbr+  20   0  146.8g   3.6g   1.0g S   8.9   5.7   8779:51 cbft.7.6.1.2024
2841004 linuxbr+  20   0  146.8g   3.3g   1.0g S   2.3   5.3   8779:51 cbft.7.6.1.2024
2841004 linuxbr+  20   0  146.8g   2.2g   1.0g S   5.6   3.5   8781:23 cbft.7.6.1.2024


###########################################################################

cb_fts_bench/cb_fts_bench -m POST -H "Content-Type: application/json" \
    -t 180s -d 5m -c 76 -f ./vindex_query_120sec_ctl.json \
    -u ${CB_USERNAME}:${CB_PASSWORD} \
    http://$CB_HOSTNAME:8094/api/bucket/target/scope/_default/index/vindex/query

RES during 5minute test
 2.1g  * idle
42.6g 76 clients  [add 20 clients  42.6 - 37.1]
                  that's 5.5g for 20 clients or 275MiB/client
		  [ add 70 clients 42.6 - 14.7
		  that's 27.9g for 20 clients or 398MiB/client


Statistics        Avg      Stdev        Max
  Reqs/sec         8.16      20.56     458.96
  Latency         9.09s      1.39s     14.10s
  HTTP codes:
    1xx - 0, 2xx - 2534, 3xx - 0, 4xx - 0, 5xx - 0
    others - 0
  Throughput:   288.95KB/s
  HTTP 429 retries            0, pct.     0.000% across 2534 reqs
  Ave Sizes
    resp.body:             519 bytes (across      2534 reqs)
    resp.body.hit[]:       375 bytes (across      2534 reqs having hits [100.00%])
  Other
    total_hits                7602, ave     3.000
    tot_hits_docreads         7602, ave     3.000
    status.total              2534
    status.failed                0
    status.successful         2534
    Test Time in Sec.      304.000


2841004 linuxbr+  20   0  154.3g   2.2g 992696 S   3.6   3.5   8897:59 cbft.7.6.1.2024