engine_testapp does not fully cleanup if exceptions thrown during testcases

Running [0040/0099]: test dcp cursor dropping...WARNING: Logging before InitGoogleLogging() is written to STDERR

W1126 05:58:56.855347 70454 HazptrDomain.h:671] Using the default inline executor for asynchronous reclamation may be susceptible to deadlock if the current thread happens to hold a resource needed by the deleter of a reclaimable object

(51905 ms) OK

Running [0041/0099]: test dcp cursor dropping (replication)...(45851 ms) OK

Running [0042/0099]: test dcp cursor dropping backfill...../kv_engine/engines/ep/tests/ep_testsuite_dcp.cc:1094 Test failed: `' (Expected `46' to be greater than or equal to `51' - Incorrect snap start seqno)

(20745 ms) FAIL

Running [0043/0099]: test dcp stream takeover...==================

WARNING: ThreadSanitizer: heap-use-after-free (virtual call vs free) (pid=64079)

  Read of size 8 at 0x7b7800097800 by thread T14:

    #0 FollyExecutorPool::TaskProxy::scheduleViaCPUPool() ../kv_engine/engines/ep/src/folly_executorpool.cc:134 (libep.so+0x00000036437a)

    #1 FollyExecutorPool::TaskProxy::timeoutExpired() ../kv_engine/engines/ep/src/folly_executorpool.cc:92 (libep.so+0x0000003733ac)

    #2 folly::HHWheelTimerBase<std::chrono::duration<long, std::ratio<1l, 1000l> > >::timeoutExpired() folly/io/async/HHWheelTimer.cpp:289 (libep.so+0x0000005f79d2)

    #3 folly::AsyncTimeout::libeventCallback(int, short, void*) folly/io/async/AsyncTimeout.cpp:176 (libep.so+0x0000005e1769)

    #4 event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1697 (libevent_core-2.1.so.7+0x0000000188e7)

    #5 folly::EventBase::loopBody(int, bool) folly/io/async/EventBase.cpp:397 (libep.so+0x0000005e79f2)

    #6 folly::EventBase::loop() folly/io/async/EventBase.cpp:315 (libep.so+0x0000005e8f5b)

    #7 folly::EventBase::loopForever() folly/io/async/EventBase.cpp:538 (libep.so+0x0000005e8f5b)

    #8 folly::IOThreadPoolExecutor::threadRun(std::shared_ptr<folly::ThreadPoolExecutor::Thread>) folly/executors/IOThreadPoolExecutor.cpp:195 (libep.so+0x0000005d1b34)

    #9 void std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(std::__invoke_memfun_deref, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:73 (libep.so+0x0000005dff84)

    #10 std::__invoke_result<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>::type std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:95 (libep.so+0x0000005dfd45)

    #11 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:467 (libep.so+0x0000005dfd45)

    #12 void std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/functional:549 (libep.so+0x0000005dfd45)

    #13 void folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) folly/Function.h:401 (libep.so+0x0000005dfd45)

    #14 folly::detail::function::FunctionTraits<void ()>::operator()() tlm/deps/folly.exploded/include/folly/Function.h:416 (libep.so+0x00000036723c)

    #15 operator() tlm/deps/folly.exploded/include/folly/executors/thread_factory/NamedThreadFactory.h:40 (libep.so+0x000000367203)

    #16 void std::__invoke_impl<void, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:60 (libep.so+0x0000003671ad)

    #17 std::__invoke_result<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>::type std::__invoke<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_result&&, (folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&)...) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/invoke.h:95 (libep.so+0x00000036711d)

    #18 _ZNSt6thread8_InvokerISt5tupleIJZN5folly18NamedThreadFactory9newThreadEONS2_8FunctionIFvvEEEEUlvE_EEE9_M_invokeIJLm0EEEEDTclsr3stdE8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:234 (libep.so+0x0000003670e5)

    #19 std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> >::operator()() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:243 (libep.so+0x0000003670a5)

    #20 std::thread::_State_impl<std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> > >::_M_run() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/thread:186 (libep.so+0x000000366f29)

    #21 execute_native_thread_routine /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 (libstdc++.so.6+0x0000000c1d3e)

  Previous write of size 8 at 0x7b7800097800 by main thread (mutexes: write M743):

    #0 free <null> (libtsan.so.0+0x00000002ac56)

    #1 folly::CPUThreadPoolExecutor::~CPUThreadPoolExecutor() folly/executors/CPUThreadPoolExecutor.cpp:124 (libep.so+0x0000005b4479)

    #2 std::default_delete<folly::CPUThreadPoolExecutor>::operator()(folly::CPUThreadPoolExecutor*) const /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/unique_ptr.h:78 (libep.so+0x00000036dc83)

    #3 std::unique_ptr<folly::CPUThreadPoolExecutor, std::default_delete<folly::CPUThreadPoolExecutor> >::reset(folly::CPUThreadPoolExecutor*) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/unique_ptr.h:371 (libep.so+0x000000363a5f)

    #4 ~FollyExecutorPool ../kv_engine/engines/ep/src/folly_executorpool.cc:705 (libep.so+0x00000035f700)

    #5 ~FollyExecutorPool ../kv_engine/engines/ep/src/folly_executorpool.cc:699 (libep.so+0x00000035f875)

    #6 ExecutorPool::shutdown() ../kv_engine/engines/ep/src/executorpool.cc:78 (libep.so+0x000000358b64)

    #7 destroy_ep_engine() ../kv_engine/engines/ep/src/ep_engine.cc:1799 (libep.so+0x0000002f543f)

    #8 shutdown_all_engines() ../kv_engine/daemon/enginemap.cc:99 (ep_testsuite_dcp+0x0000004dd204)

    #9 execute_test(test, char const*, char const*) ../kv_engine/programs/engine_testapp/engine_testapp.cc:412 (ep_testsuite_dcp+0x0000004a6857)

    #10 main ../kv_engine/programs/engine_testapp/engine_testapp.cc:629 (ep_testsuite_dcp+0x0000004a5beb)

  Mutex M743 (0x7fad63890108) created at:

    #0 pthread_mutex_lock <null> (libtsan.so.0+0x00000003bbbf)

    #1 __gthread_mutex_lock(pthread_mutex_t*) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/x86_64-linux-gnu/c++/7.5.0/bits/gthr-default.h:748 (ep_testsuite_dcp+0x000000492a93)

    #2 std::mutex::lock() /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/std_mutex.h:103 (ep_testsuite_dcp+0x000000497955)

    #3 lock_guard /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/std_mutex.h:162 (ep_testsuite_dcp+0x0000004931f6)

    #4 ExecutorPool::get() ../kv_engine/engines/ep/src/executorpool.cc:36 (libep.so+0x000000358878)

    #5 EventuallyPersistentEngine::initialize(char const*) ../kv_engine/engines/ep/src/ep_engine.cc:2138 (libep.so+0x0000002f6f38)

    #6 MockTestHarness::create_bucket(bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) ../kv_engine/programs/engine_testapp/engine_testapp.cc:212 (ep_testsuite_dcp+0x0000004a77e7)

    #7 execute_test(test, char const*, char const*) ../kv_engine/programs/engine_testapp/engine_testapp.cc:366 (ep_testsuite_dcp+0x0000004a6765)

    #8 main ../kv_engine/programs/engine_testapp/engine_testapp.cc:629 (ep_testsuite_dcp+0x0000004a5beb)

  Thread T14 'SchedulerPool0' (tid=78117, running) created by main thread at:

    #0 pthread_create <null> (libtsan.so.0+0x00000002b6f0)

    #1 __gthread_create /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:662 (libstdc++.so.6+0x0000000c1ff4)

    #2 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:163 (libstdc++.so.6+0x0000000c1ff4)

    #3 folly::NamedThreadFactory::newThread(folly::Function<void ()>&&) tlm/deps/folly.exploded/include/folly/executors/thread_factory/NamedThreadFactory.h:37 (libep.so+0x000000366232)

    #4 folly::ThreadPoolExecutor::addThreads(unsigned long) folly/executors/ThreadPoolExecutor.cpp:223 (libep.so+0x0000005d9e36)

    #5 folly::ThreadPoolExecutor::ensureActiveThreads() folly/executors/ThreadPoolExecutor.cpp:496 (libep.so+0x0000005dc91a)

    #6 folly::IOThreadPoolExecutor::getEventBase() folly/executors/IOThreadPoolExecutor.cpp:155 (libep.so+0x0000005d11d0)

    #7 FollyExecutorPool::registerTaskable(Taskable&) ../kv_engine/engines/ep/src/folly_executorpool.cc:784 (libep.so+0x00000035fe84)

    #8 KVBucket ../kv_engine/engines/ep/src/kv_bucket.cc:319 (libep.so+0x00000039feca)

    #9 EphemeralBucket ../kv_engine/engines/ep/src/ephemeral_bucket.cc:98 (libep.so+0x00000033f797)

    #10 std::_MakeUniq<EphemeralBucket>::__single_object std::make_unique<EphemeralBucket, EventuallyPersistentEngine&>(EventuallyPersistentEngine&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/unique_ptr.h:821 (libep.so+0x00000030aa14)

    #11 EventuallyPersistentEngine::makeBucket(Configuration&) ../kv_engine/engines/ep/src/ep_engine.cc:6557 (libep.so+0x0000002f7b23)

    #12 EventuallyPersistentEngine::initialize(char const*) ../kv_engine/engines/ep/src/ep_engine.cc:2207 (libep.so+0x0000002f758b)

    #13 MockTestHarness::create_bucket(bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) ../kv_engine/programs/engine_testapp/engine_testapp.cc:212 (ep_testsuite_dcp+0x0000004a77e7)

    #14 execute_test(test, char const*, char const*) ../kv_engine/programs/engine_testapp/engine_testapp.cc:366 (ep_testsuite_dcp+0x0000004a6765)

    #15 main ../kv_engine/programs/engine_testapp/engine_testapp.cc:629 (ep_testsuite_dcp+0x0000004a5beb)

SUMMARY: ThreadSanitizer: heap-use-after-free (virtual call vs free) ../kv_engine/engines/ep/src/folly_executorpool.cc:134 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()