Details
-
Bug
-
Resolution: Unresolved
-
Major
-
7.2.1
-
7.2.1-5932
-
Untriaged
-
Linux x86_64
-
0
-
Unknown
Description
Steps to Recreate:
- Create a 4 node cluster(2 KV nodes, 1 Eventing node, 1 index+n1ql node)
- Create a magma bucket with historyRetentionBytes=4294967296&historyRetentionSeconds=86400)
- Create one non default scope(scope name : scope0, Including _default scope count is 2)
- Create three collections in non default scope
- Create 1250000 documents
- Create and deploy eventing function (Eventing node: 172.23.122.142)
Function_474926663_test_cancel_timers_with_timers_being_overwritten withbindings: {'buckets': [{'alias': 'dst_bucket', 'bucket_name': 'default', 'scope_name': 'scope0', 'collection_name': 'collection1', 'access': 'rw'}, {'alias': 'dst_bucket1', 'bucket_name': 'default', 'scope_name': 'scope0', 'collection_name': 'collection2', 'access': 'rw'}], 'metadata_bucket': 'default', 'metadata_scope': '_default', 'metadata_collection': '_default', 'source_bucket': 'default', 'source_scope': 'scope0', 'source_collection': 'collection0', 'curl': []} andsettings: {'checkpoint_interval': 20000, 'cleanup_timers': False, 'dcp_stream_boundary': 'everything', 'deployment_status': False, 'description': 'Sample Description', 'log_level': 'INFO', 'skip_timer_threshold': 86400, 'sock_batch_size': 1, 'tick_duration': 5000, 'timer_processing_tick_interval': 500, 'timer_worker_pool_size': 3, 'worker_count': 3, 'processing_status': False, 'cpp_worker_thread_count': 1, 'execution_timeout': 20, 'data_chan_size': 10000, 'worker_queue_cap': 100000, 'use_memory_manager': True, 'deadline_timeout': 21, 'timer_storage_chan_size': 10000, 'dcp_gen_chan_size': 10000}[2023-09-03 10:22:07,970] - [eventing_base:435] INFO -function OnUpdate(doc,meta) {var expiry1 = new Date();expiry1.setSeconds(expiry1.getSeconds() + 30);var expiry2 = new Date();expiry2.setSeconds(expiry2.getSeconds() + 120);var expiry3 = new Date();expiry3.setSeconds(expiry3.getSeconds() + 210);var context = {docID : meta.id, random_text : "e6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh07Aumoe6cZZGHuh07Aumoe6cZZGHuh07Aumoe6"};createTimer(timerCallback, expiry1, meta.id, context);createTimer(timerCallback, expiry2, meta.id, context);createTimer(timerCallback, expiry3, meta.id, context);}function timerCallback(context) {var time_rand = random_gen();var doc_id = context.docID+'_'+time_rand;dst_bucket[doc_id] = context.random_text;}function random_gen(){var rand = Math.floor(Math.random() * 20000000) * Math.floor(Math.random() * 20000000);var time_rand = Math.round((new Date()).getTime() / 1000) + rand;return time_rand;} - Eventing stats
[2023-09-03 11:31:44,909] - [eventing_base:670] INFO - Stats for Node 172.23.122.142 is[{"dcp_feed_boundary": "everything","event_processing_stats": {"adhoc_timer_response_received": 417,"agg_messages_sent_to_worker": 1179911,"agg_queue_memory": 0,"agg_queue_memory_cap": 106954752,"agg_queue_size": 0,"agg_queue_size_cap": 300000,"agg_timer_feedback_queue_cap": 1500,"dcp_mutation": 1120128,"dcp_mutation_sent_to_worker": 1120128,"dcp_seqno_advanced": 975,"dcp_snapshot": 1061,"dcp_stream_req_counter": 1024,"dcp_streamreq": 1024,"dcp_system_event": 895,"execution_stats": 11373,"failure_stats": 11373,"latency_stats": 11373,"lcb_exception_stats": 11373,"log_level": 3,"num_processed_events": 1121998,"processed_events_size": 1135150344,"thr_count": 3,"thr_map": 3,"timer_events": 1142314,"timer_responses_received": 3360384,"v8_init": 3,"v8_load": 3,"worker_spawn_counter": 1},"events_remaining": {"dcp_backlog": 0},"execution_stats": {"agg_queue_memory": 0,"agg_queue_size": 0,"curl": {"delete": 0,"get": 0,"head": 0,"post": 0,"put": 0},"curl_success_count": 0,"dcp_delete_msg_counter": 0,"dcp_delete_parse_failure": 0,"dcp_mutation_msg_counter": 1120128,"dcp_mutation_parse_failure": 0,"enqueued_dcp_delete_msg_counter": 0,"enqueued_dcp_mutation_msg_counter": 1120128,"enqueued_timer_msg_counter": 0,"feedback_queue_size": 0,"filtered_dcp_delete_counter": 0,"filtered_dcp_mutation_counter": 0,"lcb_retry_failure": 0,"messages_parsed": 1179899,"no_op_counter": 1870,"num_processed_events": 1121998,"on_delete_failure": 0,"on_delete_success": 0,"on_update_failure": 0,"on_update_success": 1120128,"processed_events_size": 1135150344,"timer_callback_failure": 0,"timer_callback_success": 1142314,"timer_cancel_counter": 0,"timer_create_counter": 3360384,"timer_create_failure": 0,"timer_msg_counter": 1142314,"timer_responses_sent": 0,"timestamp": {"15629": "2023-09-03T18:31:44Z","15644": "2023-09-03T18:31:44Z","15995": "2023-09-03T18:31:44Z"},"uv_msg_parse_failure": 0,"uv_try_write_failure_counter": 0},"failure_stats": {"app_worker_setting_events_lost": 0,"bkt_ops_cas_mismatch_count": 0,"bucket_cache_overflow_count": 0,"bucket_op_cache_miss_count": 0,"bucket_op_exception_count": 0,"checkpoint_failure_count": 0,"curl_failure_count": 0,"curl_max_resp_size_exceeded": 0,"curl_non_200_response": 0,"curl_timeout_count": 0,"dcp_events_lost": 0,"debugger_events_lost": 0,"delete_events_lost": 0,"mutation_events_lost": 0,"n1ql_op_exception_count": 0,"timeout_count": 0,"timer_callback_missing_counter": 0,"timer_context_size_exceeded_counter": 0,"timer_events_lost": 0,"timestamp": {"15629": "2023-09-03T18:31:44Z","15644": "2023-09-03T18:31:44Z","15995": "2023-09-03T18:31:44Z"},"v8worker_events_lost": 0},"function_id": 426411847,"function_name": "Function_474926663_test_cancel_timers_with_timers_being_overwritten","function_scope": {"bucket": "default","scope": "scope0"},"gocb_creds_request_counter": 2,"internal_vb_distribution_stats": {"worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_0": "[0-341]","worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_1": "[342-682]","worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_2": "[683-1023]"},"latency_percentile_stats": {"100": 1676400,"50": 3400,"80": 3600,"90": 3800,"95": 4100,"99": 5200},"lcb_creds_request_counter": 22,"lcb_exception_stats": {},"metastore_stats": {"metastore_delete_err": 0,"metastore_deletes": 0,"metastore_not_found": 0,"metastore_scan": 0,"metastore_scan_due": 0,"metastore_scan_err": 0,"metastore_set": 0,"metastore_set_err": 0},"planner_stats": [{"host_name": "172.23.122.142:8096","start_vb": 0,"vb_count": 1024}],"vb_distribution_stats_from_metadata": {"172.23.122.142:8096": {"worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_0": "[0-341]","worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_1": "[342-682]","worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_2": "[683-1023]"}},"worker_pids": {"worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_0": 15995,"worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_1": 15629,"worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_2": 15644}}][2023-09-03 11:31:45,996] - [eventing_base:1039] INFO - Stats for Node 172.23.122.142 is[{"dcp_feed_boundary": "everything","event_processing_stats": {"adhoc_timer_response_received": 417,"agg_messages_sent_to_worker": 1179911,"agg_queue_memory": 0,"agg_queue_memory_cap": 106954752,"agg_queue_size": 0,"agg_queue_size_cap": 300000,"agg_timer_feedback_queue_cap": 1500,"dcp_mutation": 1120128,"dcp_mutation_sent_to_worker": 1120128,"dcp_seqno_advanced": 975,"dcp_snapshot": 1061,"dcp_stream_req_counter": 1024,"dcp_streamreq": 1024,"dcp_system_event": 895,"execution_stats": 11373,"failure_stats": 11373,"latency_stats": 11373,"lcb_exception_stats": 11373,"log_level": 3,"num_processed_events": 1121998,"processed_events_size": 1135150344,"thr_count": 3,"thr_map": 3,"timer_events": 1142314,"timer_responses_received": 3360384,"v8_init": 3,"v8_load": 3,"worker_spawn_counter": 1},"events_remaining": {"dcp_backlog": 0},"execution_stats": {"agg_queue_memory": 0,"agg_queue_size": 0,"curl": {"delete": 0,"get": 0,"head": 0,"post": 0,"put": 0},"curl_success_count": 0,"dcp_delete_msg_counter": 0,"dcp_delete_parse_failure": 0,"dcp_mutation_msg_counter": 1120128,"dcp_mutation_parse_failure": 0,"enqueued_dcp_delete_msg_counter": 0,"enqueued_dcp_mutation_msg_counter": 1120128,"enqueued_timer_msg_counter": 0,"feedback_queue_size": 0,"filtered_dcp_delete_counter": 0,"filtered_dcp_mutation_counter": 0,"lcb_retry_failure": 0,"messages_parsed": 1179899,"no_op_counter": 1870,"num_processed_events": 1121998,"on_delete_failure": 0,"on_delete_success": 0,"on_update_failure": 0,"on_update_success": 1120128,"processed_events_size": 1135150344,"timer_callback_failure": 0,"timer_callback_success": 1142314,"timer_cancel_counter": 0,"timer_create_counter": 3360384,"timer_create_failure": 0,"timer_msg_counter": 1142314,"timer_responses_sent": 0,"timestamp": {"15629": "2023-09-03T18:31:44Z","15644": "2023-09-03T18:31:44Z","15995": "2023-09-03T18:31:44Z"},"uv_msg_parse_failure": 0,"uv_try_write_failure_counter": 0},"failure_stats": {"app_worker_setting_events_lost": 0,"bkt_ops_cas_mismatch_count": 0,"bucket_cache_overflow_count": 0,"bucket_op_cache_miss_count": 0,"bucket_op_exception_count": 0,"checkpoint_failure_count": 0,"curl_failure_count": 0,"curl_max_resp_size_exceeded": 0,"curl_non_200_response": 0,"curl_timeout_count": 0,"dcp_events_lost": 0,"debugger_events_lost": 0,"delete_events_lost": 0,"mutation_events_lost": 0,"n1ql_op_exception_count": 0,"timeout_count": 0,"timer_callback_missing_counter": 0,"timer_context_size_exceeded_counter": 0,"timer_events_lost": 0,"timestamp": {"15629": "2023-09-03T18:31:44Z","15644": "2023-09-03T18:31:44Z","15995": "2023-09-03T18:31:44Z"},"v8worker_events_lost": 0},"function_id": 426411847,"function_name": "Function_474926663_test_cancel_timers_with_timers_being_overwritten","function_scope": {"bucket": "default","scope": "scope0"},"gocb_creds_request_counter": 2,"internal_vb_distribution_stats": {"worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_0": "[0-341]","worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_1": "[342-682]","worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_2": "[683-1023]"},"latency_percentile_stats": {"100": 1676400,"50": 3400,"80": 3600,"90": 3800,"95": 4100,"99": 5200},"lcb_creds_request_counter": 22,"lcb_exception_stats": {},"metastore_stats": {"metastore_delete_err": 0,"metastore_deletes": 0,"metastore_not_found": 0,"metastore_scan": 0,"metastore_scan_due": 0,"metastore_scan_err": 0,"metastore_set": 0,"metastore_set_err": 0},"planner_stats": [{"host_name": "172.23.122.142:8096","start_vb": 0,"vb_count": 1024}],"vb_distribution_stats_from_metadata": {"172.23.122.142:8096": {"worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_0": "[0-341]","worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_1": "[342-682]","worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_2": "[683-1023]"}},"worker_pids": {"worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_0": 15995,"worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_1": 15629,"worker_default/scope0/Function_474926663_test_cancel_timers_with_timers_being_overwritten_2": 15644}}] - Undeploy and delete the eventing function
- Observed eventing consumer crashed with segFault
Core dump was found on node 172.23.122.142(1d240fb8-951f-4123-83dd26b8-da6f8fb5.dmp))
BackTrace:
(gdb) bt full
|
#0 0x00007f7d97ac3cfc in v8::internal::MarkingBarrier::Write(v8::internal::HeapObject, v8::internal::FullHeapObjectSlot, v8::internal::HeapObject) ()
|
from /opt/couchbase/bin/../lib/libv8.so
|
No symbol table info available.
|
#1 0x00007f7d9791b214 in v8::internal::Compiler::DisposeTurbofanCompilationJob(v8::internal::Isolate*, v8::internal::TurbofanCompilationJob*, bool)
|
() from /opt/couchbase/bin/../lib/libv8.so
|
No symbol table info available.
|
#2 0x00007f7d9794a460 in v8::internal::OptimizingCompileDispatcher::FlushOutputQueue(bool) () from /opt/couchbase/bin/../lib/libv8.so
|
No symbol table info available.
|
#3 0x00007f7d9794a673 in v8::internal::OptimizingCompileDispatcher::Flush(v8::internal::BlockingBehavior) () from /opt/couchbase/bin/../lib/libv8.so
|
No symbol table info available.
|
#4 0x00007f7d97a6b5d5 in v8::internal::Heap::CollectAllAvailableGarbage(v8::internal::GarbageCollectionReason) ()
|
from /opt/couchbase/bin/../lib/libv8.so
|
No symbol table info available.
|
#5 0x00007f7d9781839c in v8::Isolate::LowMemoryNotification() () from /opt/couchbase/bin/../lib/libv8.so
|
No symbol table info available.
|
#6 0x0000000000452042 in V8Worker::ForceRunGarbageCollector (this=0x7f7d8c019000)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/v8_consumer/src/v8worker.cc:1718
|
locker = {has_lock_ = true, top_level_ = true, isolate_ = 0x7f7d8c0200a0}
|
locker = <optimized out>
|
#7 V8Worker::RouteMessage (this=0x7f7d8c019000)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/eventing/v8_consumer/src/v8worker.cc:790
|
msg = {_M_t = {<std::__uniq_ptr_impl<WorkerMessage, std::default_delete<WorkerMessage> >> = {
|
_M_t = {<std::_Tuple_impl<0, WorkerMessage*, std::default_delete<WorkerMessage> >> = {<std::_Tuple_impl<1, std::default_delete<WorkerMessage> >> = {<std::_Head_base<1, std::default_delete<WorkerMessage>, true>> = {<std::default_delete<WorkerMessage>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, WorkerMessage*, false>> = {
|
_M_head_impl = 0x7f7d74000e10}, <No data fields>}, <No data fields>}}, <No data fields>}}
|
evt = eInternal
|
val = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f7d8aff9a60 ""}, _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
|
context = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f7d8aff9a80 ""}, _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
|
callback = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f7d8aff9aa0 ""}, _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
|
#8 0x00007f7d94d39f00 in std::execute_native_thread_routine (__p=0x7f7d8c097830)
|
at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80
|
__t = <optimized out>
|
#9 0x00007f7d94aaafa3 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
|
--Type <RET> for more, q to quit, c to continue without paging--
|
No symbol table info available.
|
#10 0x00007f7d949dbeff in clone () from /lib/x86_64-linux-gnu/libc.so.6
|
No symbol table info available.
|
QE-TEST:
./testrunner -i /tmp/testexec.90558.ini -p get-cbcollect-info=True,reset_services=False,java_sdk_client=True,bucket_storage=magma,GROUP=magma_40dgm_set2,get-cbcollect-info=True,enable_cdc=True -t eventing.eventing_sanity.EventingSanity.test_cancel_timers_with_timers_being_overwritten,nodes_init=4,services_init=kv-kv-eventing-index:n1ql,dataset=default,groups=simple,reset_services=True,bucket_storage=magma,number_of_documents=1250000,bucket_size=1024,eviction_policy=fullEviction,dgm_run=True,batch_size=100000,java_sdk_client=True,GROUP=magma_40dgm_set2,use_single_bucket=True
|
Job : http://qe-jenkins1.sc.couchbase.com/job/test_suite_executor/4921/consoleFull – test_5
Job_Name: : magma_40dgm_sanity_set2_7.1_P0