Details
Description
Script to Repro
All the TAF tests are currently failing
guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/win10-bucket-ops.ini rerun=False,get-cbcollect-info=True,quota_percent=99,crash_warning=True,bucket_storage=magma,enable_dp=True,retry_get_process_num=200 -t bucket_collections.collections_rebalance.CollectionsRebalance.test_data_load_collections_with_graceful_failover_rebalance_out,nodes_init=5,nodes_failover=2,bucket_spec=multi_bucket.buckets_for_rebalance_tests_more_collections,data_load_spec=volume_test_load_with_CRUD_on_collections,data_load_stage=during,quota_percent=80,skip_validations=False,GROUP=failover_with_collection_crud'
|
We noticed some metrics like memoryTotal, memoryFree, mcdMemoryReserved, mcdMemoryAllocated are not populated in nodes/self even after 10 mins.
After initialising the node
[root@s81706 logs]# cat /opt/couchbase/VERSION.txt
|
7.0.2-6512
|
[root@s81706 logs]# curl -u Administrator:password http://127.0.0.1:8091/_prometheusMetrics | grep mem_total
|
% Total % Received % Xferd Average Speed Time Time Time Current
|
Dload Upload Total Spent Left Speed
|
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0sys_mem_total{category="system"} 3949948928
|
100 7129 0 7129 0 0 551k 0 --:--:-- --:--:-- --:--:-- 632k
|
[root@s81706 logs]# curl -u Administrator:password http://172.23.98.196:8091/nodes/self | jq | grep memoryTotal
|
% Total % Received % Xferd Average Speed Time Time Time Current
|
Dload Upload Total Spent Left Speed
|
100 2140 100 2140 0 0 404k 0 --:--:-- --:--:-- --:--:-- 522k
|
"memoryTotal": 0,
|
[root@s81706 logs]# curl -u Administrator:password http://172.23.98.196:8091/nodes/self | jq | grep uptime
|
% Total % Received % Xferd Average Speed Time Time Time Current
|
Dload Upload Total Spent Left Speed
|
100 2140 100 2140 0 0 349k 0 --:--:-- --:--:-- --:--:-- 417k
|
"uptime": "474",
|
Noticed some errors on prometheus.log.
[root@s81706 logs]# tail -f prometheus.log
|
level=debug ts=2021-08-16T07:43:19.610Z caller=scrape.go:1117 component="scrape manager" scrape_pool=general target=http://127.0.0.1:11280/_prometheusMetrics msg="Scrape failed" err="Get \"http://127.0.0.1:11280/_prometheusMetrics\": unable to read basic auth password file /opt/couchbase/var/lib/couchbase/config/prometheus_token: open /opt/couchbase/var/lib/couchbase/config/prometheus_token: no such file or directory"
|
level=debug ts=2021-08-16T07:43:19.737Z caller=scrape.go:1117 component="scrape manager" scrape_pool=general target=http://127.0.0.1:9998/_prometheusMetrics msg="Scrape failed" err="Get \"http://127.0.0.1:9998/_prometheusMetrics\": unable to read basic auth password file /opt/couchbase/var/lib/couchbase/config/prometheus_token: open /opt/couchbase/var/lib/couchbase/config/prometheus_token: no such file or directory"
|
level=debug ts=2021-08-16T07:43:23.167Z caller=scrape.go:1117 component="scrape manager" scrape_pool=kv_high_cardinality target=http://127.0.0.1:11280/_prometheusMetricsHigh msg="Scrape failed" err="Get \"http://127.0.0.1:11280/_prometheusMetricsHigh\": unable to read basic auth password file /opt/couchbase/var/lib/couchbase/config/prometheus_token: open /opt/couchbase/var/lib/couchbase/config/prometheus_token: no such file or directory"
|
level=debug ts=2021-08-16T07:43:23.920Z caller=scrape.go:1117 component="scrape manager" scrape_pool=general target=http://127.0.0.1:8091/_prometheusMetrics msg="Scrape failed" err="Get \"http://127.0.0.1:8091/_prometheusMetrics\": unable to read basic auth password file /opt/couchbase/var/lib/couchbase/config/prometheus_token: open /opt/couchbase/var/lib/couchbase/config/prometheus_token: no such file or directory"
|
level=debug ts=2021-08-16T07:43:29.610Z caller=scrape.go:1117 component="scrape manager" scrape_pool=general target=http://127.0.0.1:11280/_prometheusMetrics msg="Scrape failed" err="Get \"http://127.0.0.1:11280/_prometheusMetrics\": unable to read basic auth password file /opt/couchbase/var/lib/couchbase/config/prometheus_token: open /opt/couchbase/var/lib/couchbase/config/prometheus_token: no such file or directory"
|
level=debug ts=2021-08-16T07:43:29.737Z caller=scrape.go:1117 component="scrape manager" scrape_pool=general target=http://127.0.0.1:9998/_prometheusMetrics msg="Scrape failed" err="Get \"http://127.0.0.1:9998/_prometheusMetrics\": unable to read basic auth password file /opt/couchbase/var/lib/couchbase/config/prometheus_token: open /opt/couchbase/var/lib/couchbase/config/prometheus_token: no such file or directory"
|
level=debug ts=2021-08-16T07:43:33.167Z caller=scrape.go:1117 component="scrape manager" scrape_pool=kv_high_cardinality target=http://127.0.0.1:11280/_prometheusMetricsHigh msg="Scrape failed" err="Get \"http://127.0.0.1:11280/_prometheusMetricsHigh\": unable to read basic auth password file /opt/couchbase/var/lib/couchbase/config/prometheus_token: open /opt/couchbase/var/lib/couchbase/config/prometheus_token: no such file or directory"
|
level=debug ts=2021-08-16T07:43:33.920Z caller=scrape.go:1117 component="scrape manager" scrape_pool=general target=http://127.0.0.1:8091/_prometheusMetrics msg="Scrape failed" err="Get \"http://127.0.0.1:8091/_prometheusMetrics\": unable to read basic auth password file /opt/couchbase/var/lib/couchbase/config/prometheus_token: open /opt/couchbase/var/lib/couchbase/config/prometheus_token: no such file or directory"
|
level=debug ts=2021-08-16T07:43:39.610Z caller=scrape.go:1117 component="scrape manager" scrape_pool=general target=http://127.0.0.1:11280/_prometheusMetrics msg="Scrape failed" err="Get \"http://127.0.0.1:11280/_prometheusMetrics\": unable to read basic auth password file /opt/couchbase/var/lib/couchbase/config/prometheus_token: open /opt/couchbase/var/lib/couchbase/config/prometheus_token: no such file or directory"
|
level=debug ts=2021-08-16T07:43:39.737Z caller=scrape.go:1117 component="scrape manager" scrape_pool=general target=http://127.0.0.1:9998/_prometheusMetrics msg="Scrape failed" err="Get \"http://127.0.0.1:9998/_prometheusMetrics\": unable to read basic auth password file /opt/couchbase/var/lib/couchbase/config/prometheus_token: open /opt/couchbase/var/lib/couchbase/config/prometheus_token: no such file or directory"
|
level=debug ts=2021-08-16T07:43:43.167Z caller=scrape.go:1117 component="scrape manager" scrape_pool=kv_high_cardinality target=http://127.0.0.1:11280/_prometheusMetricsHigh msg="Scrape failed" err="Get \"http://127.0.0.1:11280/_prometheusMetricsHigh\": unable to read basic auth password file /opt/couchbase/var/lib/couchbase/config/prometheus_token: open /opt/couchbase/var/lib/couchbase/config/prometheus_token: no such file or directory"
|
level=debug ts=2021-08-16T07:43:43.920Z caller=scrape.go:1117 component="scrape manager" scrape_pool=general target=http://127.0.0.1:8091/_prometheusMetrics msg="Scrape failed" err="Get \"http://127.0.0.1:8091/_prometheusMetrics\": unable to read basic auth password file /opt/couchbase/var/lib/couchbase/config/prometheus_token: open /opt/couchbase/var/lib/couchbase/config/prometheus_token: no such file or directory"
|
^C
|
[root@s81706 logs]#
|
cbcollect_info attached.