Uploaded image for project: 'Couchbase C client library libcouchbase'
  1. Couchbase C client library libcouchbase
  2. CCBC-1619

Lost connection causes memory usage to grow indefinitely

    XMLWordPrintable

Details

    • Bug
    • Resolution: Unresolved
    • Minor
    • None
    • 3.3.10
    • None
    • Operating system Linux Debian Bullseye, Couchbase server community-7.2.2
    • 0

    Description

      We’ve ran into an issue when one of our servers written in C++ using libcouchbase 3.3.8 }}lost connection to our couchbase cluster (we are using community version {{Community Edition 6.6.0 build 7909 but it is reproducible with latest community version 7.2.2 as well)

      The server does multiple lcb_get calls with a set timeout each second. Upon losing the connection the get calls started to timeout, as expected. But simultaneously server’s memory started to grow indefinitely, going from stable 4GB to 20GB over two days, after which we manually killed it (the connection never recovered but it was a network issue on our side).

      I was able to reproduce the issue using latest community edition and official example with slight modification

       

      # on a debian(linux) host with docker 
      docker run -it --privileged --name=test-container -v /var/run/docker.sock:/var/run/docker.sock -v `pwd`:/sources debian:bullseye bash
      

      # install required packages
      apt update && apt -y install vim g++ valgrind wget gnupg git docker.io
       
      # download sources and switch to 3.3.10 tag
      cd /tmp && git clone https://github.com/couchbase/libcouchbase.git && cd libcouchbase && git checkout 3.3.10
       
      # create patch
      cat > instancepool.patch << EOF
      diff --git a/example/instancepool/main.cc b/example/instancepool/main.cc
      index 52244526..e725f10a 100644
      --- a/example/instancepool/main.cc
      +++ b/example/instancepool/main.cc
      @@ -20,15 +20,18 @@
       #include <vector>
       #include <cstring>
       #include <cstdlib>
      +#include <signal.h>
       
       using namespace lcb;
       
       extern "C" {
       static void get_callback(lcb_INSTANCE *instance, int, const lcb_RESPBASE *rb)
       {
      +    lcb_STATUS rc;
           const lcb_RESPGET *rg = reinterpret_cast< const lcb_RESPGET * >(rb);
      -    if (lcb_respget_status(rg) != LCB_SUCCESS) {
      -        fprintf(stderr, "%p: Couldn't get key", instance);
      +    rc = lcb_respget_status(rg);
      +    if (rc != LCB_SUCCESS) {
      +        fprintf(stderr, "%p: Couldn't get key: %s\n", instance, lcb_strerror_short(rc));
           } else {
               const char *key, *value;
               size_t nkey, nvalue;
      @@ -51,26 +54,36 @@ class MyPool : public Pool
               // care about
               fprintf(stderr, "Initializing %p\n", instance);
               lcb_install_callback(instance, LCB_CALLBACK_GET, get_callback);
      +        lcb_U32 tmo = 50 * 1000;
      +        lcb_cntl(instance, LCB_CNTL_SET, LCB_CNTL_OP_TIMEOUT, &tmo);
           }
       };
       
      +static int running = 1;
      +static void sigint_handler(int unused)
      +{
      +    running = 0;
      +}
      +
       extern "C" {
       static void *pthr_func(void *arg)
       {
           Pool *pool = reinterpret_cast< Pool * >(arg);
      -    lcb_CMDGET *gcmd;
      -    lcb_cmdget_create(&gcmd);
      -    lcb_cmdget_key(gcmd, "foo", 3);
       
           // Get an instance to use
           lcb_INSTANCE *instance = pool->pop();
       
      -    // Issue the command
      -    lcb_get(instance, NULL, gcmd);
      -    lcb_cmdget_destroy(gcmd);
      +    while (running) {
      +        lcb_CMDGET *gcmd;
      +        lcb_cmdget_create(&gcmd);
      +        lcb_cmdget_key(gcmd, "foo", 3);
      +        // Issue the command
      +        lcb_get(instance, NULL, gcmd);
      +        lcb_cmdget_destroy(gcmd);
       
      -    // Wait for the command to complete
      -    lcb_wait(instance, LCB_WAIT_DEFAULT);
      +        // Wait for the command to complete
      +        lcb_wait(instance, LCB_WAIT_DEFAULT);
      +    }
       
           // Release back to pool
           pool->push(instance);
      @@ -95,7 +108,7 @@ int main(int argc, char *argv[])
               lcb_createopts_credentials(options, argv[3], strlen(argv[3]), argv[2], strlen(argv[2]));
           }
       
      -    pool = new MyPool(options, 5);
      +    pool = new MyPool(options, NUM_WORKERS);
       
           err = pool->connect();
           if (err != LCB_SUCCESS) {
      @@ -103,6 +116,13 @@ int main(int argc, char *argv[])
               exit(EXIT_FAILURE);
           }
       
      +    /* setup CTRL-C handler */
      +    struct sigaction action;
      +    sigemptyset(&action.sa_mask);
      +    action.sa_handler = sigint_handler;
      +    action.sa_flags = 0;
      +    sigaction(SIGINT, &action, NULL);
      +
           for (size_t ii = 0; ii < NUM_WORKERS; ii++) {
               pthread_create(&workers[ii], NULL, pthr_func, pool);
           }
      @@ -117,3 +137,4 @@ int main(int argc, char *argv[])
           lcb_createopts_destroy(options);
           return 0;
       }
      +
      EOF
       
      # apply patch
      patch example/instancepool/main.cc instancepool.patch
       
      # add couchbase repo
      wget -q https://packages.couchbase.com/clients/c/repos/deb/couchbase.key  -O- | apt-key add -
      echo "deb https://packages.couchbase.com/clients/c/repos/deb/debian11 bullseye bullseye/main" >> /etc/apt/sources.list
      apt update && apt install -y libcouchbase3 libcouchbase-dev libcouchbase-dbg
       
      # compile modified test
      g++ -std=c++17 -o /tmp/couchbase-instancepool example/instancepool/main.cc example/instancepool/pool.cc -lcouchbase -lpthread
       
      # start and setup couchbase server
      docker run -d --name db -p 8091-8096:8091-8096 -p 11210-11211:11210-11211 couchbase:community-7.2.2
      sleep 10 # make sure server is initialized
      docker exec db couchbase-cli cluster-init --cluster-name test-cluster --cluster-username Administrator --cluster-password 123456
      sleep 10 # make sure cluster is initialized
      docker exec db couchbase-cli bucket-create --cluster localhost --username Administrator --password 123456 --bucket test-bucket --bucket-type couchbase --bucket-ramsize 512
      db_ip=$(docker inspect -f '\{{range.NetworkSettings.Networks}}\{{.IPAddress}}\{{end}}' db)
       
      # start test
      /tmp/couchbase-instancepool couchbase://${db_ip}/test-bucket 123456 Administrator
      

      After running for roughly two hours memory usage goes from initial ~20MB to ~600MB (RSS is in kilobytes).

      $ ps -p 6381 -o %mem,rss,cmd
      %MEM   RSS CMD
       1.8 609152 /tmp/couchbase-instancepool couchbase://172.17.0.3/test-bucket 123456 Administrator
      

      One thing I noticed is that when the database gets brought back up docker start db and then again back down docker stop db the memory usage does not start to grow immediately but it takes a while. It looks to me like it is some kind of a buffer (maybe retry buffer)? But it should not grow indefinitely, as it does.

      Attachments

        No reviews matched the request. Check your Options in the drop-down menu of this sections header.

        Activity

          People

            avsej Sergey Avseyev
            Kucera Martin
            Votes:
            0 Vote for this issue
            Watchers:
            1 Start watching this issue

            Dates

              Created:
              Updated:

              Gerrit Reviews

                There are no open Gerrit changes

                PagerDuty