Details
-
Bug
-
Resolution: Unresolved
-
Minor
-
None
-
3.3.10
-
None
-
Operating system Linux Debian Bullseye, Couchbase server community-7.2.2
-
0
Description
We’ve ran into an issue when one of our servers written in C++ using libcouchbase 3.3.8 }}lost connection to our couchbase cluster (we are using community version {{Community Edition 6.6.0 build 7909 but it is reproducible with latest community version 7.2.2 as well)
The server does multiple lcb_get calls with a set timeout each second. Upon losing the connection the get calls started to timeout, as expected. But simultaneously server’s memory started to grow indefinitely, going from stable 4GB to 20GB over two days, after which we manually killed it (the connection never recovered but it was a network issue on our side).
I was able to reproduce the issue using latest community edition and official example with slight modification
# on a debian(linux) host with docker
|
docker run -it --privileged --name=test-container -v /var/run/docker.sock:/var/run/docker.sock -v `pwd`:/sources debian:bullseye bash
|
# install required packages
|
apt update && apt -y install vim g++ valgrind wget gnupg git docker.io
|
|
# download sources and switch to 3.3.10 tag
|
cd /tmp && git clone https://github.com/couchbase/libcouchbase.git && cd libcouchbase && git checkout 3.3.10
|
|
# create patch
|
cat > instancepool.patch << EOF
|
diff --git a/example/instancepool/main.cc b/example/instancepool/main.cc
|
index 52244526..e725f10a 100644
|
--- a/example/instancepool/main.cc
|
+++ b/example/instancepool/main.cc
|
@@ -20,15 +20,18 @@
|
#include <vector>
|
#include <cstring>
|
#include <cstdlib>
|
+#include <signal.h>
|
|
using namespace lcb;
|
|
extern "C" {
|
static void get_callback(lcb_INSTANCE *instance, int, const lcb_RESPBASE *rb)
|
{
|
+ lcb_STATUS rc;
|
const lcb_RESPGET *rg = reinterpret_cast< const lcb_RESPGET * >(rb);
|
- if (lcb_respget_status(rg) != LCB_SUCCESS) {
|
- fprintf(stderr, "%p: Couldn't get key", instance);
|
+ rc = lcb_respget_status(rg);
|
+ if (rc != LCB_SUCCESS) {
|
+ fprintf(stderr, "%p: Couldn't get key: %s\n", instance, lcb_strerror_short(rc));
|
} else {
|
const char *key, *value;
|
size_t nkey, nvalue;
|
@@ -51,26 +54,36 @@ class MyPool : public Pool
|
// care about
|
fprintf(stderr, "Initializing %p\n", instance);
|
lcb_install_callback(instance, LCB_CALLBACK_GET, get_callback);
|
+ lcb_U32 tmo = 50 * 1000;
|
+ lcb_cntl(instance, LCB_CNTL_SET, LCB_CNTL_OP_TIMEOUT, &tmo);
|
}
|
};
|
|
+static int running = 1;
|
+static void sigint_handler(int unused)
|
+{
|
+ running = 0;
|
+}
|
+
|
extern "C" {
|
static void *pthr_func(void *arg)
|
{
|
Pool *pool = reinterpret_cast< Pool * >(arg);
|
- lcb_CMDGET *gcmd;
|
- lcb_cmdget_create(&gcmd);
|
- lcb_cmdget_key(gcmd, "foo", 3);
|
|
// Get an instance to use
|
lcb_INSTANCE *instance = pool->pop();
|
|
- // Issue the command
|
- lcb_get(instance, NULL, gcmd);
|
- lcb_cmdget_destroy(gcmd);
|
+ while (running) {
|
+ lcb_CMDGET *gcmd;
|
+ lcb_cmdget_create(&gcmd);
|
+ lcb_cmdget_key(gcmd, "foo", 3);
|
+ // Issue the command
|
+ lcb_get(instance, NULL, gcmd);
|
+ lcb_cmdget_destroy(gcmd);
|
|
- // Wait for the command to complete
|
- lcb_wait(instance, LCB_WAIT_DEFAULT);
|
+ // Wait for the command to complete
|
+ lcb_wait(instance, LCB_WAIT_DEFAULT);
|
+ }
|
|
// Release back to pool
|
pool->push(instance);
|
@@ -95,7 +108,7 @@ int main(int argc, char *argv[])
|
lcb_createopts_credentials(options, argv[3], strlen(argv[3]), argv[2], strlen(argv[2]));
|
}
|
|
- pool = new MyPool(options, 5);
|
+ pool = new MyPool(options, NUM_WORKERS);
|
|
err = pool->connect();
|
if (err != LCB_SUCCESS) {
|
@@ -103,6 +116,13 @@ int main(int argc, char *argv[])
|
exit(EXIT_FAILURE);
|
}
|
|
+ /* setup CTRL-C handler */
|
+ struct sigaction action;
|
+ sigemptyset(&action.sa_mask);
|
+ action.sa_handler = sigint_handler;
|
+ action.sa_flags = 0;
|
+ sigaction(SIGINT, &action, NULL);
|
+
|
for (size_t ii = 0; ii < NUM_WORKERS; ii++) {
|
pthread_create(&workers[ii], NULL, pthr_func, pool);
|
}
|
@@ -117,3 +137,4 @@ int main(int argc, char *argv[])
|
lcb_createopts_destroy(options);
|
return 0;
|
}
|
+
|
EOF
|
|
# apply patch
|
patch example/instancepool/main.cc instancepool.patch
|
|
# add couchbase repo
|
wget -q https://packages.couchbase.com/clients/c/repos/deb/couchbase.key -O- | apt-key add -
|
echo "deb https://packages.couchbase.com/clients/c/repos/deb/debian11 bullseye bullseye/main" >> /etc/apt/sources.list
|
apt update && apt install -y libcouchbase3 libcouchbase-dev libcouchbase-dbg
|
|
# compile modified test
|
g++ -std=c++17 -o /tmp/couchbase-instancepool example/instancepool/main.cc example/instancepool/pool.cc -lcouchbase -lpthread
|
|
# start and setup couchbase server
|
docker run -d --name db -p 8091-8096:8091-8096 -p 11210-11211:11210-11211 couchbase:community-7.2.2
|
sleep 10 # make sure server is initialized
|
docker exec db couchbase-cli cluster-init --cluster-name test-cluster --cluster-username Administrator --cluster-password 123456
|
sleep 10 # make sure cluster is initialized
|
docker exec db couchbase-cli bucket-create --cluster localhost --username Administrator --password 123456 --bucket test-bucket --bucket-type couchbase --bucket-ramsize 512
|
db_ip=$(docker inspect -f '\{{range.NetworkSettings.Networks}}\{{.IPAddress}}\{{end}}' db)
|
|
# start test
|
/tmp/couchbase-instancepool couchbase://${db_ip}/test-bucket 123456 Administrator
|
After running for roughly two hours memory usage goes from initial ~20MB to ~600MB (RSS is in kilobytes).
$ ps -p 6381 -o %mem,rss,cmd
|
%MEM RSS CMD
|
1.8 609152 /tmp/couchbase-instancepool couchbase://172.17.0.3/test-bucket 123456 Administrator
|
One thing I noticed is that when the database gets brought back up docker start db and then again back down docker stop db the memory usage does not start to grow immediately but it takes a while. It looks to me like it is some kind of a buffer (maybe retry buffer)? But it should not grow indefinitely, as it does.