{code} (gdb) info threads Id Target Id Frame 275 Thread 0x7f0d15dff700 (LWP 35582) "jemalloc_bg_thd" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 274 Thread 0x7f0d12efe700 (LWP 35586) "memcached" 0x00007f0d18fb2de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 273 Thread 0x7f0d126fd700 (LWP 35587) "memcached" 0x00007f0d18fb2de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 272 Thread 0x7f0d11efc700 (LWP 35588) "mc:check_stdin" 0x00007f0d168e4c3d in poll () from /lib64/libc.so.6 271 Thread 0x7f0d0b7fe700 (LWP 35594) "mcd:ext_auth" 0x00007f0d18fb2de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 270 Thread 0x7f0d0affd700 (LWP 35599) "mc:auditd" 0x00007f0d18fb2de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 269 Thread 0x7f0d0bfff700 (LWP 35650) "civetweb-worker" 0x00007f0d18fb554d in __lll_lock_wait () from /lib64/libpthread.so.0 268 Thread 0x7f0d10efa700 (LWP 35651) "civetweb-worker" 0x00007f0d18fb554d in __lll_lock_wait () from /lib64/libpthread.so.0 267 Thread 0x7f0d116fb700 (LWP 35652) "civetweb-master" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 266 Thread 0x7f0d0a7fc700 (LWP 35664) "mc:worker_00" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 265 Thread 0x7f0d09ffb700 (LWP 35665) "mc:worker_01" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 264 Thread 0x7f0d097fa700 (LWP 35666) "mc:worker_02" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 263 Thread 0x7f0d08ff9700 (LWP 35667) "mc:worker_03" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 262 Thread 0x7f0d03fff700 (LWP 35668) "mc:worker_04" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 261 Thread 0x7f0d037fe700 (LWP 35669) "mc:worker_05" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 260 Thread 0x7f0d02ffd700 (LWP 35670) "mc:worker_06" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 259 Thread 0x7f0d027fc700 (LWP 35671) "mc:worker_07" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 258 Thread 0x7f0d01ffb700 (LWP 35672) "mc:worker_08" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 257 Thread 0x7f0d017fa700 (LWP 35673) "mc:worker_09" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 256 Thread 0x7f0d00ff9700 (LWP 35674) "mc:worker_10" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 255 Thread 0x7f0d007f8700 (LWP 35675) "mc:worker_11" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 254 Thread 0x7f0cffff7700 (LWP 35676) "mc:worker_12" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 253 Thread 0x7f0cff7f6700 (LWP 35677) "mc:worker_13" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 252 Thread 0x7f0cfeff5700 (LWP 35678) "mc:worker_14" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 251 Thread 0x7f0cfe7f4700 (LWP 35679) "mc:worker_15" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 250 Thread 0x7f0cfdff3700 (LWP 35680) "mc:worker_16" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 249 Thread 0x7f0cfd7f2700 (LWP 35681) "mc:worker_17" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 248 Thread 0x7f0cfcff1700 (LWP 35682) "mc:worker_18" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 247 Thread 0x7f0cfc7f0700 (LWP 35683) "mc:worker_19" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 246 Thread 0x7f0cfbfef700 (LWP 35684) "mc:worker_20" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 245 Thread 0x7f0cfb7ee700 (LWP 35685) "mc:worker_21" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 244 Thread 0x7f0cfafed700 (LWP 35686) "mc:worker_22" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 243 Thread 0x7f0cfa7ec700 (LWP 35687) "mc:worker_23" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 242 Thread 0x7f0cf9feb700 (LWP 35688) "mc:worker_24" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 241 Thread 0x7f0cf97ea700 (LWP 35689) "mc:worker_25" 0x00007f0d18fb554d in __lll_lock_wait () from /lib64/libpthread.so.0 240 Thread 0x7f0cf8fe9700 (LWP 35690) "mc:worker_26" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 239 Thread 0x7f0cf87e8700 (LWP 35691) "mc:worker_27" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 238 Thread 0x7f0cf7fe7700 (LWP 35692) "mc:worker_28" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 237 Thread 0x7f0cf77e6700 (LWP 35693) "mc:worker_29" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 236 Thread 0x7f0cf5dff700 (LWP 35694) "mc:worker_30" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 235 Thread 0x7f0cf55fe700 (LWP 35695) "mc:worker_31" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 234 Thread 0x7f0cf4dfd700 (LWP 35696) "mc:worker_32" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 233 Thread 0x7f0cf45fc700 (LWP 35697) "mc:worker_33" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 232 Thread 0x7f0cf3dfb700 (LWP 35698) "mc:worker_34" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 231 Thread 0x7f0cf35fa700 (LWP 35699) "mc:worker_35" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 230 Thread 0x7f0cf2df9700 (LWP 35700) "mc:worker_36" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 229 Thread 0x7f0cf25f8700 (LWP 35701) "mc:worker_37" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 228 Thread 0x7f0cf1df7700 (LWP 35702) "mc:worker_38" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 227 Thread 0x7f0cf15f6700 (LWP 35703) "mc:worker_39" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 226 Thread 0x7f0cf0df5700 (LWP 35704) "mc:worker_40" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 225 Thread 0x7f0cf05f4700 (LWP 35705) "mc:worker_41" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 224 Thread 0x7f0cefdf3700 (LWP 35706) "mc:worker_42" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 223 Thread 0x7f0cef5f2700 (LWP 35707) "mc:worker_43" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 222 Thread 0x7f0ceedf1700 (LWP 35708) "mc:worker_44" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 221 Thread 0x7f0cee5f0700 (LWP 35709) "mc:worker_45" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 220 Thread 0x7f0ceddef700 (LWP 35710) "mc:worker_46" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 219 Thread 0x7f0ced5ee700 (LWP 35711) "mc:worker_47" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 218 Thread 0x7f0cecded700 (LWP 35712) "mc:worker_48" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 217 Thread 0x7f0cec5ec700 (LWP 35713) "mc:worker_49" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 216 Thread 0x7f0cebdeb700 (LWP 35714) "mc:worker_50" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 215 Thread 0x7f0ceb5ea700 (LWP 35715) "mc:worker_51" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 214 Thread 0x7f0ceade9700 (LWP 35716) "mc:worker_52" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 213 Thread 0x7f0cea5e8700 (LWP 35717) "mc:worker_53" 0x00007f0d18fb554d in __lll_lock_wait () from /lib64/libpthread.so.0 212 Thread 0x7f0ce9de7700 (LWP 35718) "mc:worker_54" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 211 Thread 0x7f0ce95e6700 (LWP 35719) "mc:worker_55" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 210 Thread 0x7f0ce8de5700 (LWP 35720) "mc:worker_56" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 209 Thread 0x7f0ce85e4700 (LWP 35721) "mc:worker_57" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 208 Thread 0x7f0ce7de3700 (LWP 35722) "mc:worker_58" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 207 Thread 0x7f0ce75e2700 (LWP 35723) "mc:worker_59" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 206 Thread 0x7f0ce6de1700 (LWP 35724) "mc:worker_60" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 205 Thread 0x7f0ce65e0700 (LWP 35725) "mc:worker_61" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 204 Thread 0x7f0ce5ddf700 (LWP 35726) "mc:worker_62" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 203 Thread 0x7f0ce41de700 (LWP 35727) "ReaderPool0" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 202 Thread 0x7f0ce39dd700 (LWP 35728) "ReaderPool1" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 201 Thread 0x7f0ce31dc700 (LWP 35729) "ReaderPool2" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 200 Thread 0x7f0ce11b6700 (LWP 35730) "ReaderPool3" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 199 Thread 0x7f0ce09b5700 (LWP 35731) "ReaderPool4" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 198 Thread 0x7f0cd3fff700 (LWP 35732) "ReaderPool5" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 197 Thread 0x7f0cd37fe700 (LWP 35733) "ReaderPool6" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 196 Thread 0x7f0cd2ffd700 (LWP 35734) "ReaderPool7" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 195 Thread 0x7f0cd27fc700 (LWP 35735) "ReaderPool8" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 194 Thread 0x7f0cd1ffb700 (LWP 35736) "ReaderPool9" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 193 Thread 0x7f0cd17fa700 (LWP 35737) "ReaderPool10" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 192 Thread 0x7f0cd0ff9700 (LWP 35738) "ReaderPool11" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 191 Thread 0x7f0caffff700 (LWP 35739) "ReaderPool12" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 190 Thread 0x7f0caf7fe700 (LWP 35740) "ReaderPool13" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 189 Thread 0x7f0caeffd700 (LWP 35741) "ReaderPool14" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 188 Thread 0x7f0cae7fc700 (LWP 35742) "ReaderPool15" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 187 Thread 0x7f0cadffb700 (LWP 35743) "ReaderPool16" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 186 Thread 0x7f0cad7fa700 (LWP 35744) "ReaderPool17" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 185 Thread 0x7f0cacff9700 (LWP 35745) "ReaderPool18" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 184 Thread 0x7f0c8ffff700 (LWP 35746) "ReaderPool19" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 183 Thread 0x7f0c8f7fe700 (LWP 35747) "ReaderPool20" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 182 Thread 0x7f0c8effd700 (LWP 35748) "ReaderPool21" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 181 Thread 0x7f0c8e7fc700 (LWP 35749) "ReaderPool22" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 180 Thread 0x7f0c8dffb700 (LWP 35750) "ReaderPool23" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 179 Thread 0x7f0c8d7fa700 (LWP 35751) "ReaderPool24" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 178 Thread 0x7f0c8cff9700 (LWP 35752) "ReaderPool25" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 177 Thread 0x7f0c6ffff700 (LWP 35753) "ReaderPool26" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 176 Thread 0x7f0c6f7fe700 (LWP 35754) "ReaderPool27" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 175 Thread 0x7f0c6effd700 (LWP 35755) "ReaderPool28" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 174 Thread 0x7f0c6e7fc700 (LWP 35756) "ReaderPool29" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 173 Thread 0x7f0c6dffb700 (LWP 35757) "ReaderPool30" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 172 Thread 0x7f0c6d7fa700 (LWP 35758) "ReaderPool31" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 171 Thread 0x7f0c6cff9700 (LWP 35759) "ReaderPool32" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 170 Thread 0x7f0c4ffff700 (LWP 35760) "ReaderPool33" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 169 Thread 0x7f0c4f7fe700 (LWP 35761) "ReaderPool34" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 168 Thread 0x7f0c4effd700 (LWP 35762) "ReaderPool35" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 167 Thread 0x7f0c4e7fc700 (LWP 35763) "ReaderPool36" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 166 Thread 0x7f0c4dffb700 (LWP 35764) "ReaderPool37" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 165 Thread 0x7f0c4d7fa700 (LWP 35765) "ReaderPool38" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 164 Thread 0x7f0c4cff9700 (LWP 35766) "ReaderPool39" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 163 Thread 0x7f0c2ffff700 (LWP 35767) "ReaderPool40" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 162 Thread 0x7f0c2f7fe700 (LWP 35768) "ReaderPool41" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 161 Thread 0x7f0c2effd700 (LWP 35769) "ReaderPool42" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 160 Thread 0x7f0c2e7fc700 (LWP 35770) "ReaderPool43" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 159 Thread 0x7f0c2dffb700 (LWP 35771) "ReaderPool44" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 158 Thread 0x7f0c2d7fa700 (LWP 35772) "ReaderPool45" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 157 Thread 0x7f0c2cff9700 (LWP 35773) "ReaderPool46" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 156 Thread 0x7f0c0ffff700 (LWP 35774) "ReaderPool47" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 155 Thread 0x7f0c0f7fe700 (LWP 35775) "ReaderPool48" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 154 Thread 0x7f0c0effd700 (LWP 35776) "ReaderPool49" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 153 Thread 0x7f0c0e7fc700 (LWP 35777) "ReaderPool50" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 152 Thread 0x7f0c0dffb700 (LWP 35778) "ReaderPool51" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 151 Thread 0x7f0c0d7fa700 (LWP 35779) "ReaderPool52" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 150 Thread 0x7f0c0cff9700 (LWP 35780) "ReaderPool53" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 149 Thread 0x7f0beffff700 (LWP 35781) "ReaderPool54" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 148 Thread 0x7f0bef7fe700 (LWP 35782) "ReaderPool55" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 147 Thread 0x7f0beeffd700 (LWP 35783) "ReaderPool56" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 146 Thread 0x7f0bee7fc700 (LWP 35784) "ReaderPool57" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 145 Thread 0x7f0bedffb700 (LWP 35785) "ReaderPool58" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 144 Thread 0x7f0bed7fa700 (LWP 35786) "ReaderPool59" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 143 Thread 0x7f0becff9700 (LWP 35787) "ReaderPool60" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 142 Thread 0x7f0bcffff700 (LWP 35788) "ReaderPool61" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 141 Thread 0x7f0bcf7fe700 (LWP 35789) "ReaderPool62" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 140 Thread 0x7f0bceffd700 (LWP 35790) "ReaderPool63" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 139 Thread 0x7f0bce7fc700 (LWP 35791) "ReaderPool64" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 138 Thread 0x7f0bcdffb700 (LWP 35792) "ReaderPool65" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 137 Thread 0x7f0bcd7fa700 (LWP 35793) "ReaderPool66" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 136 Thread 0x7f0bccff9700 (LWP 35794) "ReaderPool67" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 135 Thread 0x7f0baffff700 (LWP 35795) "ReaderPool68" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 134 Thread 0x7f0baf7fe700 (LWP 35796) "ReaderPool69" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 133 Thread 0x7f0baeffd700 (LWP 35797) "ReaderPool70" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 132 Thread 0x7f0bae7fc700 (LWP 35798) "ReaderPool71" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 131 Thread 0x7f0badffb700 (LWP 35799) "WriterPool0" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 130 Thread 0x7f0bad7fa700 (LWP 35800) "WriterPool1" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 129 Thread 0x7f0bacff9700 (LWP 35801) "WriterPool2" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 128 Thread 0x7f0b8ffff700 (LWP 35802) "WriterPool3" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 127 Thread 0x7f0b8f7fe700 (LWP 35803) "WriterPool4" 0x00007f0d18fb2de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 126 Thread 0x7f0b8effd700 (LWP 35804) "WriterPool5" 0x00007f0d18fb2de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 125 Thread 0x7f0b8e7fc700 (LWP 35805) "WriterPool6" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 124 Thread 0x7f0b8dffb700 (LWP 35806) "WriterPool7" 0x00007f0d18fb2de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 123 Thread 0x7f0b8d7fa700 (LWP 35807) "WriterPool8" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 122 Thread 0x7f0b8cff9700 (LWP 35808) "WriterPool9" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 121 Thread 0x7f0b6ffff700 (LWP 35809) "WriterPool10" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 120 Thread 0x7f0b6f7fe700 (LWP 35810) "WriterPool11" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 119 Thread 0x7f0b6effd700 (LWP 35811) "WriterPool12" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 118 Thread 0x7f0b6e7fc700 (LWP 35812) "WriterPool13" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 117 Thread 0x7f0b6dffb700 (LWP 35813) "WriterPool14" 0x00007f0d18fb2de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 116 Thread 0x7f0b6d7fa700 (LWP 35814) "WriterPool15" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 115 Thread 0x7f0b6cff9700 (LWP 35815) "WriterPool16" 0x00007f0d18fb554d in __lll_lock_wait () from /lib64/libpthread.so.0 114 Thread 0x7f0b4ffff700 (LWP 35816) "WriterPool17" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 113 Thread 0x7f0b4f7fe700 (LWP 35817) "WriterPool18" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 112 Thread 0x7f0b4effd700 (LWP 35818) "WriterPool19" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 111 Thread 0x7f0b4e7fc700 (LWP 35819) "WriterPool20" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 110 Thread 0x7f0b4dffb700 (LWP 35820) "WriterPool21" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 109 Thread 0x7f0b4d7fa700 (LWP 35821) "WriterPool22" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 108 Thread 0x7f0b4cff9700 (LWP 35822) "WriterPool23" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 107 Thread 0x7f0b2ffff700 (LWP 35823) "WriterPool24" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 106 Thread 0x7f0b2f7fe700 (LWP 35824) "WriterPool25" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 105 Thread 0x7f0b2effd700 (LWP 35825) "WriterPool26" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 104 Thread 0x7f0b2e7fc700 (LWP 35826) "WriterPool27" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 103 Thread 0x7f0b2dffb700 (LWP 35827) "WriterPool28" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 102 Thread 0x7f0b2d7fa700 (LWP 35828) "WriterPool29" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 101 Thread 0x7f0b2cff9700 (LWP 35829) "WriterPool30" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 100 Thread 0x7f0b0ffff700 (LWP 35830) "WriterPool31" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 99 Thread 0x7f0b0f7fe700 (LWP 35831) "WriterPool32" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 98 Thread 0x7f0b0effd700 (LWP 35832) "WriterPool33" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 97 Thread 0x7f0b0e7fc700 (LWP 35833) "WriterPool34" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 96 Thread 0x7f0b0dffb700 (LWP 35834) "WriterPool35" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 95 Thread 0x7f0b0d7fa700 (LWP 35835) "WriterPool36" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 94 Thread 0x7f0b0cff9700 (LWP 35836) "WriterPool37" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 93 Thread 0x7f0aeffff700 (LWP 35837) "WriterPool38" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 92 Thread 0x7f0aef7fe700 (LWP 35838) "WriterPool39" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 91 Thread 0x7f0aeeffd700 (LWP 35839) "WriterPool40" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 90 Thread 0x7f0aee7fc700 (LWP 35840) "WriterPool41" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 89 Thread 0x7f0aedffb700 (LWP 35841) "WriterPool42" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 88 Thread 0x7f0aed7fa700 (LWP 35842) "WriterPool43" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 87 Thread 0x7f0aecff9700 (LWP 35843) "WriterPool44" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 86 Thread 0x7f0acffff700 (LWP 35844) "WriterPool45" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 85 Thread 0x7f0acf7fe700 (LWP 35845) "WriterPool46" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 84 Thread 0x7f0aceffd700 (LWP 35846) "WriterPool47" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 83 Thread 0x7f0ace7fc700 (LWP 35847) "WriterPool48" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 82 Thread 0x7f0acdffb700 (LWP 35848) "WriterPool49" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 81 Thread 0x7f0acd7fa700 (LWP 35849) "WriterPool50" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 80 Thread 0x7f0accff9700 (LWP 35850) "WriterPool51" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 79 Thread 0x7f0aaffff700 (LWP 35851) "WriterPool52" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 78 Thread 0x7f0aaf7fe700 (LWP 35852) "WriterPool53" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 77 Thread 0x7f0aaeffd700 (LWP 35853) "WriterPool54" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 76 Thread 0x7f0aae7fc700 (LWP 35854) "WriterPool55" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 75 Thread 0x7f0aadffb700 (LWP 35855) "WriterPool56" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 74 Thread 0x7f0aad7fa700 (LWP 35856) "WriterPool57" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 73 Thread 0x7f0aacff9700 (LWP 35857) "WriterPool58" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 72 Thread 0x7f0a8ffff700 (LWP 35858) "WriterPool59" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 71 Thread 0x7f0a8f7fe700 (LWP 35859) "WriterPool60" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 70 Thread 0x7f0a8effd700 (LWP 35860) "WriterPool61" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 69 Thread 0x7f0a8e7fc700 (LWP 35861) "WriterPool62" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 68 Thread 0x7f0a8dffb700 (LWP 35862) "WriterPool63" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 67 Thread 0x7f0a8d7fa700 (LWP 35863) "WriterPool64" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 66 Thread 0x7f0a8cff9700 (LWP 35864) "WriterPool65" 0x00007f0d18fb2de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 65 Thread 0x7f0a6ffff700 (LWP 35865) "WriterPool66" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 64 Thread 0x7f0a6f7fe700 (LWP 35866) "WriterPool67" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 63 Thread 0x7f0a6effd700 (LWP 35867) "WriterPool68" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 62 Thread 0x7f0a6e7fc700 (LWP 35868) "WriterPool69" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 61 Thread 0x7f0a6dffb700 (LWP 35869) "WriterPool70" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 60 Thread 0x7f0a6d7fa700 (LWP 35870) "WriterPool71" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 59 Thread 0x7f0a6cff9700 (LWP 35871) "AuxIoPool0" 0x00007f0d18fb554d in __lll_lock_wait () from /lib64/libpthread.so.0 58 Thread 0x7f0a4ffff700 (LWP 35872) "AuxIoPool1" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 57 Thread 0x7f0a4f7fe700 (LWP 35873) "AuxIoPool2" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 56 Thread 0x7f0a4effd700 (LWP 35874) "AuxIoPool3" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 55 Thread 0x7f0a4e7fc700 (LWP 35875) "AuxIoPool4" 0x00007f0d18fb554d in __lll_lock_wait () from /lib64/libpthread.so.0 54 Thread 0x7f0a4dffb700 (LWP 35876) "AuxIoPool5" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 53 Thread 0x7f0a4d7fa700 (LWP 35877) "AuxIoPool6" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 52 Thread 0x7f0a4cff9700 (LWP 35878) "AuxIoPool7" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 51 Thread 0x7f0a2ffff700 (LWP 35879) "NonIoPool0" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 50 Thread 0x7f0a2f7fe700 (LWP 35880) "NonIoPool1" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 49 Thread 0x7f0a2effd700 (LWP 35881) "NonIoPool2" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 48 Thread 0x7f0a2e7fc700 (LWP 35882) "NonIoPool3" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 47 Thread 0x7f0a2dffb700 (LWP 35883) "NonIoPool4" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 46 Thread 0x7f0a2d7fa700 (LWP 35884) "NonIoPool5" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 45 Thread 0x7f0a2cff9700 (LWP 35885) "NonIoPool6" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 44 Thread 0x7f0a13fff700 (LWP 35886) "NonIoPool7" 0x00007f0d168e9bf9 in syscall () from /lib64/libc.so.6 43 Thread 0x7f0a137fe700 (LWP 35887) "SchedulerPool0" 0x00007f0d168efeb3 in epoll_wait () from /lib64/libc.so.6 42 Thread 0x7f0a115fd700 (LWP 35889) "jemalloc_bg_thd" 0x00007f0d18fb2de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 41 Thread 0x7f0a10dfc700 (LWP 35890) "mg:flusher:0" 0x00007f0d18fb2de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 40 Thread 0x7f0903fff700 (LWP 35891) "mg:flusher:1" 0x00007f0d18fb554d in __lll_lock_wait () from /lib64/libpthread.so.0 39 Thread 0x7f09037fe700 (LWP 35892) "mg:flusher:2" 0x00007f0d18fb554d in __lll_lock_wait () from /lib64/libpthread.so.0 38 Thread 0x7f0902ffd700 (LWP 35893) "mg:flusher:3" 0x00007f0d18fb554d in __lll_lock_wait () from /lib64/libpthread.so.0 37 Thread 0x7f09027fc700 (LWP 35894) "mg:compactor:0" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 36 Thread 0x7f0901ffb700 (LWP 35895) "mg:compactor:1" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 35 Thread 0x7f09017fa700 (LWP 35896) "mg:compactor:2" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 34 Thread 0x7f0900ff9700 (LWP 35897) "mg:compactor:3" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 33 Thread 0x7f08fafff700 (LWP 35898) "mg:compactor:4" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 32 Thread 0x7f08fa7fe700 (LWP 35899) "mg:compactor:5" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 31 Thread 0x7f08f9ffd700 (LWP 35900) "mg:compactor:6" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 30 Thread 0x7f08f97fc700 (LWP 35901) "mg:compactor:7" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 29 Thread 0x7f08f8ffb700 (LWP 35902) "mg:compactor:8" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 28 Thread 0x7f08f87fa700 (LWP 35903) "mg:compactor:9" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 27 Thread 0x7f08f7ff9700 (LWP 35904) "mg:compactor:10" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 26 Thread 0x7f08f77f8700 (LWP 35905) "mg:compactor:11" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 25 Thread 0x7f08f6ff7700 (LWP 35906) "mg:compactor:12" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 24 Thread 0x7f08f67f6700 (LWP 35907) "mg:compactor:13" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 23 Thread 0x7f08f5ff5700 (LWP 35908) "mg:compactor:14" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 22 Thread 0x7f08f57f4700 (LWP 35909) "mg:compactor:15" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 21 Thread 0x7f085dfff700 (LWP 36034) "mg:flusher:4" 0x00007f0d18fb554d in __lll_lock_wait () from /lib64/libpthread.so.0 20 Thread 0x7f085d7fe700 (LWP 36035) "mg:flusher:5" 0x00007f0d18fb2de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 19 Thread 0x7f085cffd700 (LWP 36036) "mg:flusher:6" 0x00007f0d18fb554d in __lll_lock_wait () from /lib64/libpthread.so.0 18 Thread 0x7f085c7fc700 (LWP 36037) "mg:flusher:7" 0x00007f0d18fb2de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 17 Thread 0x7f085bffb700 (LWP 36038) "mg:compactor:16" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 16 Thread 0x7f085b7fa700 (LWP 36039) "mg:compactor:17" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 15 Thread 0x7f085aff9700 (LWP 36040) "mg:compactor:18" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 14 Thread 0x7f085a7f8700 (LWP 36041) "mg:compactor:19" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 13 Thread 0x7f0859ff7700 (LWP 36042) "mg:compactor:20" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 12 Thread 0x7f08597f6700 (LWP 36043) "mg:compactor:21" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 11 Thread 0x7f0858ff5700 (LWP 36044) "mg:compactor:22" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 10 Thread 0x7f08587f4700 (LWP 36045) "mg:compactor:23" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 9 Thread 0x7f0857ff3700 (LWP 36046) "mg:compactor:24" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 8 Thread 0x7f08577f2700 (LWP 36047) "mg:compactor:25" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 7 Thread 0x7f0856ff1700 (LWP 36048) "mg:compactor:26" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 6 Thread 0x7f08567f0700 (LWP 36049) "mg:compactor:27" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 5 Thread 0x7f0855fef700 (LWP 36050) "mg:compactor:28" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 4 Thread 0x7f08557ee700 (LWP 36051) "mg:compactor:29" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 3 Thread 0x7f0854fed700 (LWP 36052) "mg:compactor:30" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 2 Thread 0x7f08547ec700 (LWP 36055) "mg:compactor:31" 0x00007f0d18fb2a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 * 1 Thread 0x7f0d19e27c40 (LWP 35581) "memcached" 0x00007f0d18fb554d in __lll_lock_wait () from /lib64/libpthread.so.0 {code} (gdb) bt #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 #1 0x00007f0d18fb0e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f0d18fb0d68 in __GI___pthread_mutex_lock (mutex=0x10e4ec0 ) at ../nptl/pthread_mutex_lock.c:78 #3 0x00000000005bc047 in __gthread_mutex_lock (__mutex=0x10e4ec0 ) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:749 #4 lock (this=0x10e4ec0 ) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:100 #5 lock_guard (__m=..., this=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:159 #6 BucketManager::forEach(std::function, void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/buckets.cc:424 #7 0x0000000000600801 in mc_gather_timing_samples () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #8 mc_time_clock_tick() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mc_time.cc:222 #9 0x0000000000600b15 in mc_time_clock_event_handler (fd=, which=, arg=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mc_time.cc:206 #10 0x00007f0d18279f58 in event_process_active_single_queue (base=base@entry=0x7f0d12f6a300, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1697 #11 0x00007f0d1827a54f in event_process_active (base=0x7f0d12f6a300) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #12 event_base_loop (base=0x7f0d12f6a300, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #13 0x0000000000bfa347 in folly::EventBase::loopBody (this=this@entry=0x7f0d12f73780, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #14 0x0000000000bfa816 in folly::EventBase::loop (this=this@entry=0x7f0d12f73780) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #15 0x0000000000bfc1d6 in folly::EventBase::loopForever (this=0x7f0d12f73780) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #16 0x00000000005659fb in memcached_main(int, char**) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/memcached.cc:1012 #17 0x00007f0d16813555 in __libc_start_main (main=0x53add0 , argc=3, argv=0x7ffd60c6b4c8, init=, fini=, rtld_fini=, stack_end=0x7ffd60c6b4b8) at ../csu/libc-start.c:266 #18 0x000000000055dec7 in _start () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/char_traits.h:322 (gdb) p *mutex $2 = {__data = {__lock = 2, __count = 0, __owner = 35651, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000C\213\000\000\001", '\000' , __align = 2} {code} (gdb) t 268 [Switching to thread 268 (Thread 0x7f0d10efa700 (LWP 35651))] #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 135 2: movl %edx, %eax (gdb) bt #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 #1 0x00007f0d18fb0e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f0d18fb0d68 in __GI___pthread_mutex_lock (mutex=0x7f08f0b67170) at ../nptl/pthread_mutex_lock.c:78 #3 0x0000000000943fd4 in __gthread_mutex_lock (__mutex=0x7f08f0b67170) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:749 #4 lock (this=0x7f08f0b67170) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:100 #5 lock (this=0x7f0d10ee3f40) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_lock.h:138 #6 unique_lock (__m=..., this=0x7f0d10ee3f40) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_lock.h:68 #7 GetStats (cacheDuration=..., magmaStats=..., this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_stats.cc:352 #8 magma::Magma::GetStats(magma::Magma::MagmaStats&, std::chrono::duration >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_stats.cc:368 #9 0x00000000008908e0 in MagmaMemoryTrackingProxy::GetStats(std::chrono::duration >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-memory-tracking-proxy.cc:248 #10 0x000000000086eca0 in MagmaKVStore::getAggrDbFileInfo (this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:3130 #11 0x000000000081f0f5 in EPBucket::getFileStats(BucketStatCollector const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvshard.h:72 #12 0x00000000006b310b in EventuallyPersistentEngine::doEngineStatsLowCardinality(BucketStatCollector const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:3128 #13 0x00000000006bde51 in EventuallyPersistentEngine::get_prometheus_stats(BucketStatCollector const&, cb::prometheus::Cardinality) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:402 #14 0x00000000005a64bd in operator() (bucket=..., __closure=0x7f0d10ee4390) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stats.cc:228 #15 __invoke_impl&, Bucket&, void*> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #16 __invoke_r&, Bucket&, void*> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:113 #17 std::_Function_handler::_M_invoke(std::_Any_data const&, Bucket&, void*&&) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291 #18 0x00000000005bc0fd in operator() (__args#1=, __args#0=..., this=0x7f0d10ee4390) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #19 BucketManager::forEach(std::function, void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/buckets.cc:446 #20 0x00000000005a3a1e in server_prometheus_stats(StatCollector const&, cb::prometheus::Cardinality) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #21 0x0000000000abb3da in operator() (__args#1=, __args#0=..., this=0x7f0d12f57720) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #22 cb::prometheus::MetricServer::KVCollectable::Collect (this=0x7f0d12f57710) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/statistics/prometheus.cc:122 #23 0x0000000000ad36bb in prometheus::detail::CollectMetrics (collectables=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/pull/src/metrics_collector.cc:18 #24 0x0000000000ad26f8 in prometheus::detail::MetricsHandler::handleGet (this=0x7f0d13dfe880, conn=0x1acd698) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/pull/src/handler.cc:129 #25 0x0000000000ac0897 in CivetServer::requestHandler (conn=0x1acd698, cbdata=0x7f0d13dfe880) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/CivetServer.cpp:152 #26 0x0000000000acd63d in handle_request (conn=conn@entry=0x1acd698) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:14657 #27 0x0000000000ace699 in process_new_connection (conn=0x1acd698) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18673 #28 worker_thread_run (conn=0x1acd698) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19090 #29 worker_thread (thread_func_param=0x1acd698) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19145 #30 0x00007f0d18faeea5 in start_thread (arg=0x7f0d10efa700) at pthread_create.c:307 #31 0x00007f0d168ef8dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 {code} {code} (gdb) f 2 #2 0x00007f0d18fb0d68 in __GI___pthread_mutex_lock (mutex=0x7f08f0b67170) at ../nptl/pthread_mutex_lock.c:78 78 LLL_MUTEX_LOCK (mutex); (gdb) p *mutex $3 = {__data = {__lock = 2, __count = 0, __owner = 35717, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000\205\213\000\000\001", '\000' , __align = 2} {code} (gdb) t 213 [Switching to thread 213 (Thread 0x7f0cea5e8700 (LWP 35717))] #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 135 2: movl %edx, %eax (gdb) bt #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 #1 0x00007f0d18fb0e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f0d18fb0d68 in __GI___pthread_mutex_lock (mutex=0x7f08f0b665b8) at ../nptl/pthread_mutex_lock.c:78 #3 0x0000000000960d4e in __gthread_mutex_lock (__mutex=0x7f08f0b665b8) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:749 #4 lock (this=0x7f08f0b665b8) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:100 #5 lock_guard (__m=..., this=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:159 #6 magma::KVStoreSet::GetAllKVStores() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore_set.cc:49 #7 0x000000000094074a in magma::Magma::Impl::getStats(magma::Magma::MagmaStats&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_stats.cc:181 #8 0x0000000000944010 in GetStats (cacheDuration=..., magmaStats=..., this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_stats.cc:360 #9 magma::Magma::GetStats(magma::Magma::MagmaStats&, std::chrono::duration >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_stats.cc:368 #10 0x00000000008908e0 in MagmaMemoryTrackingProxy::GetStats(std::chrono::duration >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-memory-tracking-proxy.cc:248 #11 0x000000000086eca0 in MagmaKVStore::getAggrDbFileInfo (this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:3130 #12 0x000000000081f0f5 in EPBucket::getFileStats(BucketStatCollector const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvshard.h:72 #13 0x00000000006b310b in EventuallyPersistentEngine::doEngineStatsLowCardinality(BucketStatCollector const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:3128 #14 0x00000000006bc98b in doEngineStats (collector=..., this=0x7f0d10000000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:4970 #15 EventuallyPersistentEngine::getStats(void const*, std::basic_string_view >, std::basic_string_view >, std::function >, std::basic_string_view >, void const*)> const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:4970 #16 0x00000000006bd569 in EventuallyPersistentEngine::get_stats(CookieIface const&, std::basic_string_view >, std::basic_string_view >, std::function >, std::basic_string_view >, void const*)> const&) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #17 0x0000000000588a98 in bucket_get_stats(Cookie&, std::basic_string_view >, cb::sized_buffer, std::function >, std::basic_string_view >, void const*)> const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:372 #18 0x0000000000646171 in stat_all_stats(std::__cxx11::basic_string, std::allocator > const&, Cookie&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/stats_context.cc:459 #19 0x0000000000647143 in StatsCommandContext::doStats() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/stats_context.cc:695 #20 0x00000000006482a0 in StatsCommandContext::step() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/stats_context.cc:617 #21 0x000000000064c091 in SteppableCommandContext::drive() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:36 #22 0x00000000005fb2ab in Cookie::doExecute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/mcbp/protocol/header.h:110 #23 0x00000000005fc650 in Cookie::execute (this=this@entry=0x7f08529a8980, useStartTime=useStartTime@entry=true) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:168 #24 0x00000000005dc201 in Connection::executeCommandPipeline() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.h:469 #25 0x00000000005dc493 in Connection::executeCommandsCallback() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:797 #26 0x00000000005dcb1b in Connection::rw_callback(bufferevent*, void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:946 #27 0x00007f0d18270d8e in bufferevent_run_deferred_callbacks_unlocked (cb=, arg=0x7f08529a8c00) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 #28 0x00007f0d18279d01 in event_process_active_single_queue (base=base@entry=0x7f0d13512800, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 #29 0x00007f0d1827a54f in event_process_active (base=0x7f0d13512800) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #30 event_base_loop (base=0x7f0d13512800, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #31 0x0000000000bfa347 in folly::EventBase::loopBody (this=this@entry=0x7f0d13635220, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #32 0x0000000000bfa816 in folly::EventBase::loop (this=this@entry=0x7f0d13635220) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #33 0x0000000000bfc1d6 in folly::EventBase::loopForever (this=this@entry=0x7f0d13635220) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #34 0x00000000005abc79 in worker_libevent (arg=0x7f0d13635210) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:115 #35 0x0000000000b86049 in run (this=0x7f0d13570df0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #36 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #37 0x00007f0d18faeea5 in start_thread (arg=0x7f0cea5e8700) at pthread_create.c:307 #38 0x00007f0d168ef8dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 (gdb) f 2 #2 0x00007f0d18fb0d68 in __GI___pthread_mutex_lock (mutex=0x7f08f0b665b8) at ../nptl/pthread_mutex_lock.c:78 78 LLL_MUTEX_LOCK (mutex); (gdb) p *mutex $4 = {__data = {__lock = 2, __count = 0, __owner = 35893, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000\065\214\000\000\001", '\000' , __align = 2} (gdb) t 38 [Switching to thread 38 (Thread 0x7f0902ffd700 (LWP 35893))] #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 135 2: movl %edx, %eax (gdb) bt #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 #1 0x00007f0d18fb0e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f0d18fb0d68 in __GI___pthread_mutex_lock (mutex=0x7f08efdf8ef0) at ../nptl/pthread_mutex_lock.c:78 #3 0x0000000000976a57 in __gthread_mutex_lock (__mutex=0x7f08efdf8ef0) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:749 #4 lock (this=0x7f08efdf8ef0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:100 #5 lock (this=0x7f0902fea980) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_lock.h:138 #6 unique_lock (__m=..., this=0x7f0902fea980) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_lock.h:68 #7 magma::WAL::Trim(magma::WALOffset&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/wal/wal.cc:558 #8 0x0000000000960287 in magma::KVStoreSet::Trim(magma::WAL*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore_set.cc:104 #9 0x0000000000952c4e in magma::Magma::Impl::tryWALTrim() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_write.cc:86 #10 0x00000000009332f4 in magma::Magma::Impl::tryWriteCacheFlush(magma::BlockingMode, magma::FlushMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1134 #11 0x00000000009335ab in magma::Magma::Impl::heartbeatCallback() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1252 #12 0x00000000009ae934 in operator() (this=0x7efb67252e10) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #13 magma::TimedTask::Complete(bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:144 #14 0x00000000009b0121 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:161 #15 0x0000000000b86049 in run (this=0x7f0a11679e30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #16 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #17 0x00007f0d18faeea5 in start_thread (arg=0x7f0902ffd700) at pthread_create.c:307 #18 0x00007f0d168ef8dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 (gdb) f 2 #2 0x00007f0d18fb0d68 in __GI___pthread_mutex_lock (mutex=0x7f08efdf8ef0) at ../nptl/pthread_mutex_lock.c:78 78 LLL_MUTEX_LOCK (mutex); (gdb) p *mutex $5 = {__data = {__lock = 2, __count = 0, __owner = 35864, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000\030\214\000\000\001", '\000' , __align = 2} (gdb) t 66 [Switching to thread 66 (Thread 0x7f0a8cff9700 (LWP 35864))] #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 238 62: movq %rax, %r14 (gdb) bt #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00000000009e15d1 in __gthread_cond_timedwait (__abs_timeout=0x7f0a8cfe6160, __mutex=, __cond=0x7f08efc92268) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=..., this=0x7f08efc92268) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7f08efc92268) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:126 #4 operator() (__closure=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:349 #5 magma::LSMTree::tryLevel0BlockingCompaction() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:364 #6 0x00000000009e9e72 in add (autoFlush=false, rawVal=..., rawKey=..., op=magma::Update, this=0x7f08efc92000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:316 #7 magma::LSMTree::Write(magma::Operation, magma::Slice const&, magma::Slice const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:1491 #8 0x000000000097e08e in magma::KVStore::writeDocument(magma::Magma::WriteOperation const&, magma::WAL*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:827 #9 0x000000000097e105 in magma::KVStore::write(magma::Magma::WriteOperation const&, magma::WAL*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:630 #10 0x000000000098c719 in magma::KVStore::WriteDocs(magma::WAL*, std::vector > const&, std::function, std::function > const*> ()>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/write.cc:162 #11 0x0000000000952f55 in magma::Magma::Impl::WriteDocs(unsigned short, std::vector > const&, unsigned int, std::function, std::function > const*> ()>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #12 0x0000000000953102 in magma::Magma::WriteDocs(unsigned short, std::vector > const&, unsigned int, std::function, std::function > const*> ()>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #13 0x0000000000890ef4 in MagmaMemoryTrackingProxy::WriteDocs(unsigned short, std::vector > const&, unsigned int, std::function, std::function > const*> ()>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #14 0x0000000000878d9f in MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #15 0x00000000008718ff in MagmaKVStore::commit(std::unique_ptr >, VB::Commit&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:711 #16 0x000000000081b605 in EPBucket::commit(KVStoreIface&, std::unique_ptr >, VB::Commit&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:927 #17 0x00000000008242dd in EPBucket::flushVBucket_UNLOCKED(LockedVBucketPtr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:799 #18 0x0000000000824daf in EPBucket::flushVBucket(Vbid) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:376 #19 0x00000000006d7920 in Flusher::flushVB (this=0x7f086197d400) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:285 #20 0x00000000006d8270 in Flusher::step(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:200 #21 0x0000000000a9eeb2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #22 0x0000000000a9c0c5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f0a8cfe7840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #23 0x0000000000bf0c60 in operator() (this=0x7f0a8cfe7840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #24 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f0d134fa400, thread=..., task=task@entry=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #25 0x0000000000bd8a1a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f0d134fa400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #26 0x0000000000bf3c19 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #27 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #28 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #29 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #30 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #31 0x0000000000a9bd54 in operator() (this=0x7f0d135b28c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #32 operator() (__closure=0x7f0d135b28c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #33 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #34 0x00007f0d171a6d40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #35 0x00007f0d18faeea5 in start_thread (arg=0x7f0a8cff9700) at pthread_create.c:307 #36 0x00007f0d168ef8dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 ^^^ Looks like {{magma::LSMTree::tryLevel0BlockingCompaction()}} is waiting on a condition variable while holding a lock which other threads (in a chain) are waiting on.