FTS in OOM killed during vector query execution while rebalance in of a FTS node is in progress.

Continuing the test steps after MB-60367.

1. Start a upsert workload on the vector dataset.
2. Start the vector search query workload.
3. Rebalance in 1 FTS node.
4. Rebalance failed:

   Service 'fts' exited with status 137. Restarting. Messages:

   2024-01-14T16:48:22.028-08:00 [WARN] slow-query: index: default0VolumeCollection0_fts_idx_0, username: <ud>Administrator</ud>, query: <ud>{"explain": false, "query": {"match_none": {}}, "knn": [{"k": 81, "field": "embedding", "vector": [-0.1532357633113861, 0.09714271873235703, -0.015255401842296124, 0.0014427612768486142, -0.051508743315935135, -0.013760652393102646, 0.13659968972206116, -0.01676359586417675, -0.03323420509696007, 0.015979617834091187, -0.034563418477773666, -0.004722922574728727, 0.05042849853634834, -0.06200791150331497, 0.04221874102950096, -0.04523662477731705, 0.1044069454073906, 0.058997396379709244, 0.018156206235289574, -0.01217223983258009, -0.0017111002234742045, 0.027848761528730392, -0.001038730377331376, -0.02771732211112976, -0.049730122089385986, 0.03407028689980507, 0.04497525468468666, 0.07821899652481079, -0.10925346612930298, -0.05864231288433075, 0.00914513599127531, 0.06607069820165634, -0.006559897214174271, -0.01773953065276146, 0.006863467860966921, -0.04208246245980263, 0.06746848672628403, -0.03329624980688095, 0.03923872485756874, 0.040610309690237045, -0.031008007004857063, -0.03715309128165245, 0.023955797776579857, 0.0681983083486557, -0.011269493959844112, 0.03625469654798508, -0.015557312406599522, 0.07525966316461563, 0.02264609932899475, -0.00802669022232294, -0.021981703117489815, 0.008252207189798355, 0.015652211382985115, -0.005530690308660269, 0.04364286735653877, 0.009230307303369045, -0.02800285629928112, 0.08929259330034256, -0.049451347440481186, -0.005328403320163488, -0.00992639735341072, -0.029102951288223267, -0.0012280564988031983, 0.019656110554933548, 0.050372034311294556, -0.043558068573474884, -0.08231816440820694, 0.055039312690496445, 0.07643634080886841, 0.0036558329593390226, 0.06146061792969704, 0.025193357840180397, -0.08570852130651474, 0.010102253407239914, -0.08176872134208679, 0.03039771318435669, 0.010424287058413029, -0.011713755317032337, 0.033890001475811005, 0.006701852660626173, -0.051375988870859146, -0.0675872340798378, 0.005752857308834791, 0.036956366151571274, -0.045188743621110916, 0.022820737212896347, -0.014464432373642921, 0.009506474249064922, -0.07676229625940323, 0.10952489078044891, 0.02550036646425724, -0.022084688767790794, 0.03964032977819443, 0.07931387424468994, -0.05841226875782013, -0.08194500207901001, -0.04171460494399071, -0.001163782668299973, -0.11401306837797165, 0.1459265947341919, 0.05458208918571472, -0.028099652379751205, 0.049068156629800797, 0.007238900754600763, 0.0608602836728096, -0.04052015021443367, 0.049552563577890396, -0.005940900184214115, 0.06300592422485352, 0.01935259997844696, -0.017878366634249687, -0.029412442818284035, -0.06309542059898376, 0.027772823348641396, -0.062184348702430725, -0.10927717387676239, 0.058282990008592606, 0.03460659086704254, 0.07309655845165253, -0.12171868979930878, 0.032147523015737534, 0.013793699443340302, -0.03490915149450302, -0.046247247606515884, -0.10618803650140762, 0.060655511915683746, -0.037279825657606125, 1.693100696674827e-33, -0.013669361360371113, 0.0380571149289608, 0.050770845264196396, 0.07908374071121216, -0.025283610448241234, 0.031766366213560104, -0.07119151204824448, -0.06343656033277512, 0.008863192051649094, -0.023004520684480667, -0.07430744916200638, 0.061904359608888626, -0.03812870755791664, 0.03340080752968788, 0.029980208724737167, -0.0711774230003357, -0.0035858354531228542, -0.013294802978634834, 0.026182549074292183, 0.006483437493443489, -0.03350825607776642, 0.08922872692346573, -0.02749931812286377, -0.026827048510313034, -0.03613464534282684, 0.05344226211309433, 0.013629469089210033, -0.030062297359108925, -0.030334923416376114, 0.07674256712198257, 0.03039201907813549, -0.07745086401700974, 0.059044480323791504, 0.05224985256791115, -0.09222563356161118, -0.012091332115232944, -0.030485041439533234, -0.009976435452699661, 0.006760692689567804, -0.014821397140622139, 0.0010539713548496366, -0.07754788547754288, -0.031729646027088165, 0.09490281343460083, -0.019450657069683075, 0.06799931824207306, 0.0245744027197361, 0.06622873246669769, 0.004537050146609545, -0.005310537293553352, 0.033925049006938934, -0.003687912132591009, 0.025014612823724747, -0.08001348376274109, -0.016473019495606422, 0.02014918439090252, 0.023491285741329193, 0.050746090710163116, 0.017686955630779266, 0.013396629132330418, 0.08632367104291916, 0.10380695760250092, 0.05835892632603645, -0.023695718497037888, 0.0440942607820034, -0.017138132825493813, 0.005447338800877333, -0.02530292607843876, -0.046734169125556946, -0.003861081786453724, 0.029780982062220573, -0.024179693311452866, 0.14674071967601776, -0.011350343003869057, 0.05610913038253784, -0.011173906736075878, 0.0255624670535326, 0.03631764277815819, -0.02453436516225338, -0.07209672033786774, 0.0896839126944542, 0.0018507697386667132, -0.007590876426547766, -0.04122848063707352, 0.02461990900337696, 0.049541275948286057, -0.007850066758692265, -0.03757968544960022, 0.007693325635045767, 0.033796507865190506, -0.06723442673683167, -0.06373376399278641, 0.015377652831375599, 0.06503858417272568, 0.007729274220764637, -2.2802118194395805e-33, 0.009543107822537422, 0.006078552454710007, 0.04719727486371994, 0.06392505019903183, 0.07875461876392365, -0.003934917040169239, 0.01045963168144226, 0.047348227351903915, -0.015584373846650124, -0.014840339310467243, -0.024088334292173386, -0.06850629299879074, -0.025490662083029747, -0.00013166473945602775, -0.010777758434414864, 0.0690285861492157, 0.0078062983229756355, -0.023142889142036438, -0.02567482739686966, -0.025337601080536842, -0.04813055321574211, -0.03982861340045929, 0.0046525318175554276, 0.08781091868877411, -0.08961372077465057, -0.007523194886744022, 0.08806423097848892, 0.02667953446507454, -0.0871104747056961, 0.031334515661001205, 0.0806582123041153, -0.05747680738568306, -0.06101369857788086, 0.04056935012340546, 0.008044255897402763, -0.0006353388889692724, -0.07244446873664856, 0.07028216868638992, 0.02414727210998535, -0.01981714554131031, -0.062499865889549255, -0.04017156362533569, -0.023808032274246216, 0.05579796060919762, 0.07039710134267807, -0.04654493182897568, -0.06234891712665558, -0.005819410085678101, -0.012494257651269436, 0.06630362570285797, -0.07788270711898804, 0.03631013259291649, -0.07160547375679016, 0.05665665119886398, -0.045940183103084564, -0.002001811284571886, -0.03837627172470093, 0.11123734712600708, -0.025155553594231606, 0.1371668577194214, -0.039525795727968216, -0.022020580247044563, -0.0028985936660319567, 0.043597713112831116, 0.027052586898207664, 0.011224846355617046, -0.06181048974394798, 0.020280610769987106, -0.035110585391521454, 0.0021666893735527992, 0.107246033847332, -0.013413223437964916, -0.0017486161086708307, 0.011942926794290543, 0.04372606799006462, -0.005398747976869345, -0.020180033519864082, -0.002524079754948616, -0.04437475651502609, 0.01059596799314022, -0.05008605495095253, -0.01813572831451893, -0.04952732473611832, 0.1399821639060974, -0.04125896468758583, 0.022869182750582695, 0.06447166204452515, 0.019831102341413498, -0.056845735758543015, 0.020090434700250626, 0.04218870773911476, 0.0093674436211586, -0.04074461758136749, 0.05535006523132324, -0.007829846814274788, -1.575893016081409e-08, 0.055964261293411255, 0.04529762268066406, -0.007236041594296694, -0.0398864708840847, 0.01366643700748682, -0.027396442368626595, -0.037537574768066406, -0.015145562589168549, 0.055120084434747696, 0.005751824472099543, -0.03774073347449303, 0.00021789790480397642, -0.04631110280752182, 0.05414523556828499, -0.057029105722904205, 0.022428203374147415, -0.02996310032904148, 0.0035385575611144304, -0.03230442479252815, -0.04252418130636215, 0.055335938930511475, 0.0021434403024613857, 0.06230168417096138, 0.002741805976256728, -0.0340535007417202, 0.0030299632344394922, -0.028569785878062248, -0.06721355766057968, 0.13134683668613434, 0.04685114696621895, -0.02844269759953022, 0.04710744693875313, -0.0817728191614151, -0.042162150144577026, -0.02781064435839653, -0.03840893134474754, -0.044650107622146606, -0.015970326960086823, 0.019539283588528633, -0.0663946345448494, -0.06434950232505798, -0.15714871883392334, 0.05136037617921829, 0.003484113607555628, -0.032287925481796265, -0.048607878386974335, 0.014190939255058765, 0.1020023375749588, -0.09765326231718063, 0.06647109985351562, -0.00326799089089036, 0.053818296641111374, -0.01935722678899765, 0.019074751064181328, -0.027122022584080696, -0.008249766193330288, -0.05595899745821953, 0.10079681873321533, -0.02835501916706562, 0.00140860874671489, -0.05233750492334366, -0.1068955734372139, -0.06287917494773865, -0.06271270662546158]}]}</ud>, resultset bytes: 6779, duration: 13.173651846s, err: <nil> -- rest.(*QueryHandler).ServeHTTP() at rest_index.go:388

   2024-01-14T16:48:22.570-08:00 [WARN] grpc_client: Query() returned error from host: 172.23.107.237:9130, err: grpc_client: query got status code: 504, resp: &bleve.SearchResult{Status:(*bleve.SearchStatus)(0xc1e35b6d00), Request:(*bleve.SearchRequest)(0xc01eeacd20), Hits:search.DocumentMatchCollection(nil), Total:0x0, Cost:0x0, MaxScore:0, Took:0, Facets:search.FacetResults(nil)}, err: rpc error: code = DeadlineExceeded desc = context deadline exceeded -- cbft.(*GrpcClient).SearchInContext.func1() at grpc_client.go:153