Probably the log callback happens after something has been deallocated
# gdb /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/build/bin/unit-tests /tmp/*.55774.* --batch -ex 'thread apply all bt' 2>&1
[New LWP 55774]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `/home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_'.
Program terminated with signal 11, Segmentation fault.
#0 0x00007f894111b848 in main_arena () from /lib64/libc.so.6
Thread 1 (Thread 0x7f8942ed3f80 (LWP 55774)):
#0 0x00007f894111b848 in main_arena () from /lib64/libc.so.6
#1 0x000000000052e3e2 in lcb_log (settings=<optimized out>, subsys=subsys@entry=0x6194bd "SSL", severity=severity@entry=0, srcfile=srcfile@entry=0x627738 "/home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/src/ssl/ssl_common.c", srcline=srcline@entry=260, fmt=fmt@entry=0x6277b8 "<%s:%s> sock=%p: ST(0x%x). %s. R(0x%x) %s (%s)") at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/src/logging.c:163
#2 0x00000000005bcdfb in log_callback (ssl=0x209de10, where=16388, ret=256) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/src/ssl/ssl_common.c:260
#3 0x00007f894240a92a in ssl3_read_bytes () from /lib64/libssl.so.10
#4 0x00007f89424076d4 in ssl3_read_internal () from /lib64/libssl.so.10
#5 0x00000000005f49e4 in schedule_wants (cs=cs@entry=0x20bf000) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/src/ssl/ssl_c.c:242
#6 0x00000000005f5505 in read_callback (sd=<optimized out>, nr=506, arg=0x20bf000) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/src/ssl/ssl_c.c:228
#7 0x00007f893f947a4e in read_cb (stream=0x208b9c8, nread=<optimized out>, buf=<optimized out>) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/plugins/io/libuv/plugin-libuv.c:464
#8 0x00007f893f7325b4 in uv__read () from /lib64/libuv.so.1
#9 0x00007f893f732e8c in uv__stream_io () from /lib64/libuv.so.1
#10 0x00007f893f73a8f3 in uv__io_poll () from /lib64/libuv.so.1
#11 0x00007f893f727d3d in uv_run () from /lib64/libuv.so.1
#12 0x00000000005b1b4e in lcb_wait (instance=0x1fc5f30, flags=<optimized out>) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/src/wait.cc:108
#13 0x000000000052850a in drop_collection (instance=0x1fc5f30, scope="sCollectionDropMiss614597378", collection="cCollectionDropMiss1124698465", wait=wait@entry=true) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/tests/iotests/testutil.cc:449
#14 0x0000000000436398 in CollectionUnitTest_testDroppedCollection_Test::TestBody (this=<optimized out>) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/tests/iotests/t_collections.cc:384
#15 0x00007f8942ab2633 in HandleSehExceptionsInMethodIfSupported<testing::Test, void> (location=0x7f8942ab4fd7 "the test body", method=<optimized out>, object=<optimized out>) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2443
#16 testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void> (object=object@entry=0x1f4ce70, method=<optimized out>, location=location@entry=0x7f8942ab4fd7 "the test body") at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2479
#17 0x00007f8942aa2ea7 in testing::Test::Run (this=this@entry=0x1f4ce70) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2518
#18 0x00007f8942aa2fa8 in testing::TestInfo::Run (this=0x1f305f0) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2693
#19 0x00007f8942aa3055 in testing::TestCase::Run (this=0x1f2ff10) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2811
#20 0x00007f8942aa76c8 in testing::internal::UnitTestImpl::RunAllTests (this=this@entry=0x1f2e710) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:5178
#21 0x00007f8942aa799c in HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (location=<optimized out>, method=<optimized out>, object=<optimized out>) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2443
#22 HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (location=0x7f8942ab76c0 "auxiliary test code (environments or event listeners)", method=(bool (testing::internal::UnitTestImpl::*)(testing::internal::UnitTestImpl * const)) 0x7f8942aa7490 <testing::internal::UnitTestImpl::RunAllTests()>, object=0x1f2e710) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2479
#23 testing::UnitTest::Run (this=<optimized out>) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:4790
#24 0x0000000000412704 in RUN_ALL_TESTS () at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/include/gtest/gtest.h:2341
#25 main (argc=1, argv=0x7fff546d9148) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/tests/unit_tests.cc:45
https://sdk.jenkins.couchbase.com/job/c-cpp/job/lcb/job/lcb-lnx-scripted-build-pipeline/1640/
Probably the log callback happens after something has been deallocated
# gdb /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/build/bin/unit-tests /tmp/*.55774.* --batch -ex 'thread apply all bt' 2>&1 [New LWP 55774] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by `/home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_'. Program terminated with signal 11, Segmentation fault. #0 0x00007f894111b848 in main_arena () from /lib64/libc.so.6 Thread 1 (Thread 0x7f8942ed3f80 (LWP 55774)): #0 0x00007f894111b848 in main_arena () from /lib64/libc.so.6 #1 0x000000000052e3e2 in lcb_log (settings=<optimized out>, subsys=subsys@entry=0x6194bd "SSL", severity=severity@entry=0, srcfile=srcfile@entry=0x627738 "/home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/src/ssl/ssl_common.c", srcline=srcline@entry=260, fmt=fmt@entry=0x6277b8 "<%s:%s> sock=%p: ST(0x%x). %s. R(0x%x) %s (%s)") at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/src/logging.c:163 #2 0x00000000005bcdfb in log_callback (ssl=0x209de10, where=16388, ret=256) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/src/ssl/ssl_common.c:260 #3 0x00007f894240a92a in ssl3_read_bytes () from /lib64/libssl.so.10 #4 0x00007f89424076d4 in ssl3_read_internal () from /lib64/libssl.so.10 #5 0x00000000005f49e4 in schedule_wants (cs=cs@entry=0x20bf000) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/src/ssl/ssl_c.c:242 #6 0x00000000005f5505 in read_callback (sd=<optimized out>, nr=506, arg=0x20bf000) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/src/ssl/ssl_c.c:228 #7 0x00007f893f947a4e in read_cb (stream=0x208b9c8, nread=<optimized out>, buf=<optimized out>) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/plugins/io/libuv/plugin-libuv.c:464 #8 0x00007f893f7325b4 in uv__read () from /lib64/libuv.so.1 #9 0x00007f893f732e8c in uv__stream_io () from /lib64/libuv.so.1 #10 0x00007f893f73a8f3 in uv__io_poll () from /lib64/libuv.so.1 #11 0x00007f893f727d3d in uv_run () from /lib64/libuv.so.1 #12 0x00000000005b1b4e in lcb_wait (instance=0x1fc5f30, flags=<optimized out>) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/src/wait.cc:108 #13 0x000000000052850a in drop_collection (instance=0x1fc5f30, scope="sCollectionDropMiss614597378", collection="cCollectionDropMiss1124698465", wait=wait@entry=true) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/tests/iotests/testutil.cc:449 #14 0x0000000000436398 in CollectionUnitTest_testDroppedCollection_Test::TestBody (this=<optimized out>) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/tests/iotests/t_collections.cc:384 #15 0x00007f8942ab2633 in HandleSehExceptionsInMethodIfSupported<testing::Test, void> (location=0x7f8942ab4fd7 "the test body", method=<optimized out>, object=<optimized out>) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2443 #16 testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void> (object=object@entry=0x1f4ce70, method=<optimized out>, location=location@entry=0x7f8942ab4fd7 "the test body") at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2479 #17 0x00007f8942aa2ea7 in testing::Test::Run (this=this@entry=0x1f4ce70) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2518 #18 0x00007f8942aa2fa8 in testing::TestInfo::Run (this=0x1f305f0) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2693 #19 0x00007f8942aa3055 in testing::TestCase::Run (this=0x1f2ff10) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2811 #20 0x00007f8942aa76c8 in testing::internal::UnitTestImpl::RunAllTests (this=this@entry=0x1f2e710) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:5178 #21 0x00007f8942aa799c in HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (location=<optimized out>, method=<optimized out>, object=<optimized out>) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2443 #22 HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (location=0x7f8942ab76c0 "auxiliary test code (environments or event listeners)", method=(bool (testing::internal::UnitTestImpl::*)(testing::internal::UnitTestImpl * const)) 0x7f8942aa7490 <testing::internal::UnitTestImpl::RunAllTests()>, object=0x1f2e710) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:2479 #23 testing::UnitTest::Run (this=<optimized out>) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/src/gtest.cc:4790 #24 0x0000000000412704 in RUN_ALL_TESTS () at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/contrib/gtest-1.8.1/include/gtest/gtest.h:2341 #25 main (argc=1, argv=0x7fff546d9148) at /home/couchbase/jenkins/workspace/c-cpp/lcb/lcb-lnx-scripted-build-pipeline/ws_centos7/libcouchbase/tests/unit_tests.cc:45