/Users/admin/testrunner/venv/bin/python "/Applications/PyCharm CE.app/Contents/plugins/python-ce/helpers/pydev/pydevd.py" --multiproc --qt-support=auto --client 127.0.0.1 --port 63695 --file /Users/admin/testrunner/testrunner.py -i b/resources/tmp.ini -p gsi_type=plasma,java_sdk_client=False,bucket_size=200,reset_services=True -c conf/gsi/collection-concurrent-indexes.conf pydev debugger: process 44448 is connecting Connected to pydev debugger (build 202.7660.27) ./lib/remote/remote_util.py:4234: SyntaxWarning: "is not" with a literal. Did you mean "!="? if command_options is not '': lib/membase/api/rest_client.py:71: SyntaxWarning: "is not" with a literal. Did you mean "!="? while progress is not -1 and progress < percentage and retry < retry_count: lib/remote/remote_util.py:4234: SyntaxWarning: "is not" with a literal. Did you mean "!="? if command_options is not '': INFO:root:__main__ *** TestRunner *** INFO:__main__:TestRunner: parsing args... INFO:__main__:Checking arguments... INFO:__main__:Conf filename: conf/gsi/collection-concurrent-indexes.conf INFO:__main__:Test prefix: gsi.collections_concurrent_indexes.ConcurrentIndexes INFO:__main__:TestRunner: start... INFO:__main__:Global Test input params: INFO:__main__: Number of tests initially selected before GROUP filters: 1 INFO:__main__:--> Running test: gsi.collections_concurrent_indexes.ConcurrentIndexes.test_index_creation_with_rebalanced_out_assigned_index_node,default_bucket=false,force_clean=true,defer_build=True,GROUP=P1,num_of_indexes=10,nodes_init=3,reset_services=True,services_init=kv:n1ql:index-index-index,num_retries_for_failed_index=2 INFO:__main__:Logs folder: /Users/admin/testrunner/logs/testrunner-20-Nov-10_12-58-36/test_1 {'bucket_size': '200', 'cluster_name': 'tmp', 'conf_file': 'conf/gsi/collection-concurrent-indexes.conf', 'gsi_type': 'plasma', 'ini': 'b/resources/tmp.ini', 'java_sdk_client': 'False', 'num_nodes': 4, 'reset_services': 'True', 'spec': 'collection-concurrent-indexes'} Logs will be stored at /Users/admin/testrunner/logs/testrunner-20-Nov-10_12-58-36/test_1 ./testrunner -i b/resources/tmp.ini -p gsi_type=plasma,java_sdk_client=False,bucket_size=200,reset_services=True -t gsi.collections_concurrent_indexes.ConcurrentIndexes.test_index_creation_with_rebalanced_out_assigned_index_node,default_bucket=false,force_clean=true,defer_build=True,GROUP=P1,num_of_indexes=10,nodes_init=3,reset_services=True,services_init=kv:n1ql:index-index-index,num_retries_for_failed_index=2 Test Input params: {'default_bucket': 'false', 'force_clean': 'true', 'defer_build': 'True', 'GROUP': 'P1', 'num_of_indexes': '10', 'nodes_init': '3', 'reset_services': 'True', 'services_init': 'kv:n1ql:index-index-index', 'num_retries_for_failed_index': '2', 'ini': 'b/resources/tmp.ini', 'cluster_name': 'tmp', 'spec': 'collection-concurrent-indexes', 'conf_file': 'conf/gsi/collection-concurrent-indexes.conf', 'gsi_type': 'plasma', 'java_sdk_client': 'False', 'bucket_size': '200', 'num_nodes': 4, 'case_number': 1, 'logs_folder': '/Users/admin/testrunner/logs/testrunner-20-Nov-10_12-58-36/test_1'} Run before suite setup for gsi.collections_concurrent_indexes.ConcurrentIndexes.test_index_creation_with_rebalanced_out_assigned_index_node lib/tasks/task.py:300: SyntaxWarning: "is" with a literal. Did you mean "=="? if bucket_params['bucket_priority'] is None or bucket_params['bucket_priority'].lower() is 'low': ./lib/membase/api/rest_client.py:71: SyntaxWarning: "is not" with a literal. Did you mean "!="? while progress is not -1 and progress < percentage and retry < retry_count: pytests/basetestcase.py:1432: SyntaxWarning: "is not" with a literal. Did you mean "!="? if self.stats_monitor is not '': pytests/basetestcase.py:1434: SyntaxWarning: "is not" with a literal. Did you mean "!="? if self.warmup_stats_monitor is not '': lib/couchbase_helper/tuq_helper.py:209: SyntaxWarning: "is" with a literal. Did you mean "=="? if status is "any": lib/couchbase_helper/tuq_helper.py:219: SyntaxWarning: "is" with a literal. Did you mean "=="? if status is "any": suite_setUp (gsi.collections_concurrent_indexes.ConcurrentIndexes) ... -->before_suite_name:gsi.collections_concurrent_indexes.ConcurrentIndexes.suite_setUp,suite: ]> 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [rest_client.get_nodes_version] Node version in cluster 7.0.0-3649-enterprise 2020-11-10 12:58:37 | ERROR | MainProcess | MainThread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [rest_client.get_nodes_versions] Node versions in cluster [] 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [basetestcase.setUp] ============== basetestcase setup was started for test #1 suite_setUp============== 2020-11-10 12:58:37 | ERROR | MainProcess | MainThread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default/ body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.101, nothing to delete 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.102, nothing to delete 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.103, nothing to delete 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.104, nothing to delete 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.101:8091 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.101:8091 is running 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.102:8091 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.102:8091 is running 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.103:8091 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.103:8091 is running 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.104:8091 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.104:8091 is running 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [collections_concurrent_indexes.tearDown] ============== ConcurrentIndexes tearDown has started ============== 2020-11-10 12:58:37 | ERROR | MainProcess | MainThread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 12:58:37 | ERROR | MainProcess | MainThread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [basetestcase.get_nodes_from_services_map] cannot find service node index in cluster 2020-11-10 12:58:37 | ERROR | MainProcess | MainThread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [basetestcase.print_cluster_stats] ------- Cluster statistics ------- 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [basetestcase.print_cluster_stats] --- End of cluster statistics --- 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [basetestcase.tearDown] ============== basetestcase cleanup was started for test #1 suite_setUp ============== 2020-11-10 12:58:37 | ERROR | MainProcess | MainThread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default/ body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.101, nothing to delete 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.102, nothing to delete 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.103, nothing to delete 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.104, nothing to delete 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.101:8091 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.101:8091 is running 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.102:8091 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.102:8091 is running 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.103:8091 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.103:8091 is running 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.104:8091 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.104:8091 is running 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [basetestcase.tearDown] ============== basetestcase cleanup was finished for test #1 suite_setUp ============== 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [basetestcase.tearDown] closing all ssh connections 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [basetestcase.tearDown] closing all memcached connections 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 12:58:37 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 12:58:38 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:58:38 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:58:38 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 12:58:38 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 12:58:38 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:58:38 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:58:38 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.101 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 12:58:38 | INFO | MainProcess | MainThread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 12:58:38 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl stop couchbase-server.service 2020-11-10 12:58:40 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:58:40 | INFO | MainProcess | MainThread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 12:58:40 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 12:58:45 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 12:58:45 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:58:45 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 12:58:45 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:58:45 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 12:58:45 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 12:58:45 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:58:46 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:58:46 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.101 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 12:58:46 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 12:58:46 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 12:58:46 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 12:58:46 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl start couchbase-server.service 2020-11-10 12:58:46 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:58:46 | INFO | MainProcess | MainThread | [remote_util.sleep] 10.112.205.101:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 12:58:51 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 12:58:51 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:58:51 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Couchbase server status: [' Process: 2395 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 12:58:51 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 12:58:51 | INFO | MainProcess | MainThread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.101: with pid 2476 2020-11-10 12:58:51 | INFO | MainProcess | MainThread | [basetestcase.start_server] Couchbase started 2020-11-10 12:58:51 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 12:58:51 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 12:58:51 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:58:51 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:58:51 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 12:58:51 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 12:58:51 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:58:52 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:58:52 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.102 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 12:58:52 | INFO | MainProcess | MainThread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 12:58:52 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl stop couchbase-server.service 2020-11-10 12:58:54 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:58:54 | INFO | MainProcess | MainThread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 12:58:54 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.102 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl start couchbase-server.service 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:58:59 | INFO | MainProcess | MainThread | [remote_util.sleep] 10.112.205.102:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 12:59:04 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 12:59:04 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:04 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Couchbase server status: [' Process: 2384 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 12:59:04 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 12:59:04 | INFO | MainProcess | MainThread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.102: with pid 2465 2020-11-10 12:59:04 | INFO | MainProcess | MainThread | [basetestcase.start_server] Couchbase started 2020-11-10 12:59:04 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 12:59:05 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 12:59:05 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:59:05 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:59:05 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 12:59:05 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 12:59:05 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:59:05 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:59:05 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.103 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 12:59:05 | INFO | MainProcess | MainThread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 12:59:05 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl stop couchbase-server.service 2020-11-10 12:59:07 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:07 | INFO | MainProcess | MainThread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 12:59:07 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 12:59:12 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 12:59:12 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:12 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 12:59:12 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:12 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 12:59:12 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 12:59:12 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:59:13 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:59:13 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.103 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 12:59:13 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 12:59:13 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 12:59:13 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 12:59:13 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl start couchbase-server.service 2020-11-10 12:59:13 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:13 | INFO | MainProcess | MainThread | [remote_util.sleep] 10.112.205.103:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 12:59:18 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 12:59:18 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:18 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Couchbase server status: [' Process: 2361 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 12:59:18 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 12:59:18 | INFO | MainProcess | MainThread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.103: with pid 2442 2020-11-10 12:59:18 | INFO | MainProcess | MainThread | [basetestcase.start_server] Couchbase started 2020-11-10 12:59:18 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 12:59:18 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 12:59:18 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:59:18 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:59:18 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 12:59:18 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 12:59:19 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:59:19 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:59:19 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.104 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 12:59:19 | INFO | MainProcess | MainThread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 12:59:19 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl stop couchbase-server.service 2020-11-10 12:59:21 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:21 | INFO | MainProcess | MainThread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 12:59:21 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.104 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl start couchbase-server.service 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:26 | INFO | MainProcess | MainThread | [remote_util.sleep] 10.112.205.104:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 12:59:31 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 12:59:31 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:31 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Couchbase server status: [' Process: 2360 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 12:59:31 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 12:59:32 | INFO | MainProcess | MainThread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.104: with pid 2441 2020-11-10 12:59:32 | INFO | MainProcess | MainThread | [basetestcase.start_server] Couchbase started 2020-11-10 12:59:32 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 10 secs. ... Cluster instance shutdown with force 2020-11-10 12:59:42 | INFO | MainProcess | MainThread | [collections_concurrent_indexes.tearDown] ============== ConcurrentIndexes tearDown has completed ============== 2020-11-10 12:59:42 | INFO | MainProcess | MainThread | [basetestcase.setUp] initializing cluster 2020-11-10 12:59:42 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 12:59:42 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 12:59:42 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:59:42 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:59:42 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 12:59:42 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 12:59:42 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:59:42 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:59:42 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.101 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 12:59:43 | INFO | MainProcess | MainThread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 12:59:43 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl stop couchbase-server.service 2020-11-10 12:59:44 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:44 | INFO | MainProcess | MainThread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 12:59:44 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 12:59:49 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 12:59:49 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:49 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 12:59:49 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:49 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 12:59:49 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 12:59:50 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:59:50 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:59:50 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.101 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 12:59:50 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 12:59:50 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 12:59:50 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 12:59:50 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl start couchbase-server.service 2020-11-10 12:59:50 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:50 | INFO | MainProcess | MainThread | [remote_util.sleep] 10.112.205.101:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 12:59:55 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 12:59:55 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:55 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Couchbase server status: [' Process: 2793 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 12:59:55 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 12:59:55 | INFO | MainProcess | MainThread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.101: with pid 2874 2020-11-10 12:59:55 | INFO | MainProcess | MainThread | [basetestcase.start_server] Couchbase started 2020-11-10 12:59:55 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 12:59:55 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 12:59:55 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:59:56 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:59:56 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 12:59:56 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 12:59:56 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 12:59:56 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 12:59:56 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.102 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 12:59:56 | INFO | MainProcess | MainThread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 12:59:56 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl stop couchbase-server.service 2020-11-10 12:59:58 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 12:59:58 | INFO | MainProcess | MainThread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 12:59:58 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.102 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl start couchbase-server.service 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:03 | INFO | MainProcess | MainThread | [remote_util.sleep] 10.112.205.102:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:00:08 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:00:08 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:08 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Couchbase server status: [' Process: 2789 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:00:08 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:00:08 | INFO | MainProcess | MainThread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.102: with pid 2870 2020-11-10 13:00:08 | INFO | MainProcess | MainThread | [basetestcase.start_server] Couchbase started 2020-11-10 13:00:08 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:00:08 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:00:09 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:00:09 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:00:09 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:00:09 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:00:09 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:00:09 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:00:09 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.103 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:00:09 | INFO | MainProcess | MainThread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:00:09 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl stop couchbase-server.service 2020-11-10 13:00:11 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:11 | INFO | MainProcess | MainThread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:00:11 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:00:16 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:00:16 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:16 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:00:16 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:16 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:00:16 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:00:16 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:00:17 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:00:17 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.103 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:00:17 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:00:17 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:00:17 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:00:17 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl start couchbase-server.service 2020-11-10 13:00:17 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:17 | INFO | MainProcess | MainThread | [remote_util.sleep] 10.112.205.103:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:00:22 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:00:22 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:22 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Couchbase server status: [' Process: 2763 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:00:22 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:00:22 | INFO | MainProcess | MainThread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.103: with pid 2844 2020-11-10 13:00:22 | INFO | MainProcess | MainThread | [basetestcase.start_server] Couchbase started 2020-11-10 13:00:22 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:00:22 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:00:22 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:00:22 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:00:22 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:00:22 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:00:22 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:00:23 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:00:23 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.104 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:00:23 | INFO | MainProcess | MainThread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:00:23 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl stop couchbase-server.service 2020-11-10 13:00:25 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:25 | INFO | MainProcess | MainThread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:00:25 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.104 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl start couchbase-server.service 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:30 | INFO | MainProcess | MainThread | [remote_util.sleep] 10.112.205.104:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:00:35 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:00:35 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:00:35 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Couchbase server status: [' Process: 2761 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:00:35 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:00:35 | INFO | MainProcess | MainThread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.104: with pid 2842 2020-11-10 13:00:35 | INFO | MainProcess | MainThread | [basetestcase.start_server] Couchbase started 2020-11-10 13:00:35 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 10 secs. ... 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [task.execute] server: ip:10.112.205.101 port:8091 ssh_username:root, nodes/self 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [task.execute] {'uptime': '53', 'memoryTotal': 2962632704, 'memoryFree': 2588770304, 'mcdMemoryReserved': 2260, 'mcdMemoryAllocated': 2260, 'status': 'healthy', 'hostname': '10.112.205.101:8091', 'clusterCompatibility': 458752, 'clusterMembership': 'active', 'version': '7.0.0-3649-enterprise', 'os': 'x86_64-unknown-linux-gnu', 'ports': [], 'availableStorage': [], 'storage': [], 'memoryQuota': 980, 'moxi': 11211, 'memcached': 11210, 'id': 'ns_1@10.112.205.101', 'ip': '10.112.205.101', 'rest_username': '', 'rest_password': '', 'port': '8091', 'services': ['kv'], 'storageTotalRam': 2825, 'curr_items': 0} 2020-11-10 13:00:46 | ERROR | MainProcess | Cluster_Thread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [task.execute] quota for index service will be 256 MB 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [task.execute] set index quota to node 10.112.205.101 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [rest_client.set_service_memoryQuota] pools/default params : indexMemoryQuota=256 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster_memoryQuota] pools/default params : memoryQuota=1258 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [rest_client.init_node_services] --> init_node_services(Administrator,password,10.112.205.101,8091,['kv', 'n1ql', 'index']) 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [rest_client.init_node_services] /node/controller/setupServices params on 10.112.205.101: 8091:hostname=10.112.205.101&user=Administrator&password=password&services=kv%2Cn1ql%2Cindex 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> in init_cluster...Administrator,password,8091 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] settings/web params on 10.112.205.101:8091:port=8091&username=Administrator&password=password 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> status:True 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:00:46 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] command executed with root but got an error [' % Total % Received % Xferd Average Speed Time Time Time Current', ' Dload Upload Total Spent Left Speed', '', ' 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0', '100 43 100 2 100 41 12 259 --:--:-- --:--:-- --:--:-- 261'] ... 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [remote_util.enable_diag_eval_on_non_local_hosts] ['ok'] 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.101:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.101:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [rest_client.set_indexer_storage_mode] settings/indexes params : storageMode=plasma 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [task.execute] server: ip:10.112.205.102 port:8091 ssh_username:root, nodes/self 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [task.execute] {'uptime': '38', 'memoryTotal': 1930829824, 'memoryFree': 1562624000, 'mcdMemoryReserved': 1473, 'mcdMemoryAllocated': 1473, 'status': 'healthy', 'hostname': '10.112.205.102:8091', 'clusterCompatibility': 458752, 'clusterMembership': 'active', 'version': '7.0.0-3649-enterprise', 'os': 'x86_64-unknown-linux-gnu', 'ports': [], 'availableStorage': [], 'storage': [], 'memoryQuota': 256, 'moxi': 11211, 'memcached': 11210, 'id': 'ns_1@10.112.205.102', 'ip': '10.112.205.102', 'rest_username': '', 'rest_password': '', 'port': '8091', 'services': ['kv'], 'storageTotalRam': 1841, 'curr_items': 0} 2020-11-10 13:00:47 | ERROR | MainProcess | Cluster_Thread | [rest_client._http_request] GET http://10.112.205.102:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster_memoryQuota] pools/default params : memoryQuota=986 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> in init_cluster...Administrator,password,8091 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] settings/web params on 10.112.205.102:8091:port=8091&username=Administrator&password=password 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> status:True 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 13:00:47 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] command executed with root but got an error [' % Total % Received % Xferd Average Speed Time Time Time Current', ' Dload Upload Total Spent Left Speed', '', ' 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0', '100 43 100 2 100 41 12 266 --:--:-- --:--:-- --:--:-- 267'] ... 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [remote_util.enable_diag_eval_on_non_local_hosts] ['ok'] 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.102:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.102:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [rest_client.set_indexer_storage_mode] settings/indexes params : storageMode=plasma 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [task.execute] server: ip:10.112.205.103 port:8091 ssh_username:root, nodes/self 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [task.execute] {'uptime': '28', 'memoryTotal': 1930829824, 'memoryFree': 1552334848, 'mcdMemoryReserved': 1473, 'mcdMemoryAllocated': 1473, 'status': 'healthy', 'hostname': '10.112.205.103:8091', 'clusterCompatibility': 458752, 'clusterMembership': 'active', 'version': '7.0.0-3649-enterprise', 'os': 'x86_64-unknown-linux-gnu', 'ports': [], 'availableStorage': [], 'storage': [], 'memoryQuota': 256, 'moxi': 11211, 'memcached': 11210, 'id': 'ns_1@10.112.205.103', 'ip': '10.112.205.103', 'rest_username': '', 'rest_password': '', 'port': '8091', 'services': ['kv'], 'storageTotalRam': 1841, 'curr_items': 0} 2020-11-10 13:00:48 | ERROR | MainProcess | Cluster_Thread | [rest_client._http_request] GET http://10.112.205.103:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster_memoryQuota] pools/default params : memoryQuota=986 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> in init_cluster...Administrator,password,8091 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] settings/web params on 10.112.205.103:8091:port=8091&username=Administrator&password=password 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> status:True 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:00:48 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] command executed with root but got an error [' % Total % Received % Xferd Average Speed Time Time Time Current', ' Dload Upload Total Spent Left Speed', '', ' 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0', '100 43 100 2 100 41 13 267 --:--:-- --:--:-- --:--:-- 267'] ... 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [remote_util.enable_diag_eval_on_non_local_hosts] ['ok'] 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.103:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.103:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [rest_client.set_indexer_storage_mode] settings/indexes params : storageMode=plasma 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [task.execute] server: ip:10.112.205.104 port:8091 ssh_username:root, nodes/self 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [task.execute] {'uptime': '13', 'memoryTotal': 1930829824, 'memoryFree': 1579511808, 'mcdMemoryReserved': 1473, 'mcdMemoryAllocated': 1473, 'status': 'healthy', 'hostname': '10.112.205.104:8091', 'clusterCompatibility': 458752, 'clusterMembership': 'active', 'version': '7.0.0-3649-enterprise', 'os': 'x86_64-unknown-linux-gnu', 'ports': [], 'availableStorage': [], 'storage': [], 'memoryQuota': 256, 'moxi': 11211, 'memcached': 11210, 'id': 'ns_1@10.112.205.104', 'ip': '10.112.205.104', 'rest_username': '', 'rest_password': '', 'port': '8091', 'services': ['kv'], 'storageTotalRam': 1841, 'curr_items': 0} 2020-11-10 13:00:49 | ERROR | MainProcess | Cluster_Thread | [rest_client._http_request] GET http://10.112.205.104:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster_memoryQuota] pools/default params : memoryQuota=986 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> in init_cluster...Administrator,password,8091 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] settings/web params on 10.112.205.104:8091:port=8091&username=Administrator&password=password 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> status:True 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:00:49 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:00:50 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:00:50 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2020-11-10 13:00:50 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] command executed with root but got an error [' % Total % Received % Xferd Average Speed Time Time Time Current', ' Dload Upload Total Spent Left Speed', '', ' 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0', ' 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0', '100 43 100 2 100 41 13 267 --:--:-- --:--:-- -- ... 2020-11-10 13:00:50 | INFO | MainProcess | Cluster_Thread | [remote_util.enable_diag_eval_on_non_local_hosts] ['ok'] 2020-11-10 13:00:50 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.104:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:00:50 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.104:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:00:50 | INFO | MainProcess | Cluster_Thread | [rest_client.set_indexer_storage_mode] settings/indexes params : storageMode=plasma 2020-11-10 13:00:50 | INFO | MainProcess | MainThread | [basetestcase.add_built_in_server_user] **** add built-in 'cbadminbucket' user to node 10.112.205.101 **** 2020-11-10 13:00:50 | ERROR | MainProcess | MainThread | [rest_client._http_request] DELETE http://10.112.205.101:8091/settings/rbac/users/local/cbadminbucket body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"User was not found."' auth: Administrator:password 2020-11-10 13:00:50 | INFO | MainProcess | MainThread | [internal_user.delete_user] Exception while deleting user. Exception is -b'"User was not found."' 2020-11-10 13:00:50 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:00:55 | INFO | MainProcess | MainThread | [basetestcase.add_built_in_server_user] **** add 'admin' role to 'cbadminbucket' user **** 2020-11-10 13:00:55 | INFO | MainProcess | MainThread | [basetestcase.setUp] done initializing cluster 2020-11-10 13:00:56 | INFO | MainProcess | Cluster_Thread | [task.add_nodes] adding node 10.112.205.102:8091 to cluster 2020-11-10 13:00:56 | INFO | MainProcess | Cluster_Thread | [rest_client.add_node] adding remote node @10.112.205.102:8091 to this cluster @10.112.205.101:8091 2020-11-10 13:00:58 | INFO | MainProcess | Cluster_Thread | [task.add_nodes] adding node 10.112.205.103:8091 to cluster 2020-11-10 13:00:58 | INFO | MainProcess | Cluster_Thread | [rest_client.add_node] adding remote node @10.112.205.103:8091 to this cluster @10.112.205.101:8091 2020-11-10 13:01:00 | INFO | MainProcess | Cluster_Thread | [rest_client.rebalance] rebalance params : {'knownNodes': 'ns_1@10.112.205.101,ns_1@10.112.205.102,ns_1@10.112.205.103', 'ejectedNodes': '', 'user': 'Administrator', 'password': 'password'} 2020-11-10 13:01:00 | INFO | MainProcess | Cluster_Thread | [rest_client.rebalance] rebalance operation started 2020-11-10 13:01:00 | INFO | MainProcess | Cluster_Thread | [rest_client._rebalance_status_and_progress] rebalance percentage : 0.00 % 2020-11-10 13:01:00 | INFO | MainProcess | Cluster_Thread | [task.check] Rebalance - status: running, progress: 0.00% 2020-11-10 13:01:10 | INFO | MainProcess | Cluster_Thread | [task.check] Rebalance - status: none, progress: 100.00% 2020-11-10 13:01:10 | INFO | MainProcess | Cluster_Thread | [task.check] rebalancing was completed with progress: 100% in 10.04590106010437 sec 2020-11-10 13:01:10 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:01:10 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:01:10 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed with root but got an error [' % Total % Received % Xferd Average Speed Time Time Time Current', ' Dload Upload Total Spent Left Speed', '', ' 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0', ' 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0', '100 43 100 2 100 41 13 267 --:--:-- --:--:-- -- ... 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [remote_util.enable_diag_eval_on_non_local_hosts] ['ok'] 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [basetestcase.enable_diag_eval_on_non_local_hosts] Enabled diag/eval for non-local hosts from 10.112.205.101 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [basetestcase.setUp] ============== basetestcase setup was finished for test #1 suite_setUp ============== 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [basetestcase.print_cluster_stats] ------- Cluster statistics ------- 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [basetestcase.print_cluster_stats] 10.112.205.101:8091 => {'services': ['index', 'kv', 'n1ql'], 'cpu_utilization': 8.526315789473685, 'mem_free': 2464284672, 'mem_total': 2962632704, 'swap_mem_used': 0, 'swap_mem_total': 1107292160} 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [basetestcase.print_cluster_stats] 10.112.205.102:8091 => {'services': ['index'], 'cpu_utilization': 4.979253112033195, 'mem_free': 1545056256, 'mem_total': 1930829824, 'swap_mem_used': 0, 'swap_mem_total': 1107292160} 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [basetestcase.print_cluster_stats] 10.112.205.103:8091 => {'services': ['index'], 'cpu_utilization': 4.968944099378882, 'mem_free': 1547464704, 'mem_total': 1930829824, 'swap_mem_used': 0, 'swap_mem_total': 1107292160} 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [basetestcase.print_cluster_stats] --- End of cluster statistics --- 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [newtuq.setUp] Initial status of 10.112.205.101 cluster is healthy 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [newtuq.setUp] current status of 10.112.205.101 is healthy 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [newtuq.setUp] Initial status of 10.112.205.102 cluster is healthy 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [newtuq.setUp] current status of 10.112.205.102 is healthy 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [newtuq.setUp] Initial status of 10.112.205.103 cluster is healthy 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [newtuq.setUp] current status of 10.112.205.103 is healthy 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [newtuq.setUp] Initial status of 10.112.205.104 cluster is healthy 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [newtuq.setUp] current status of 10.112.205.104 is healthy 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [basetestcase.get_nodes_from_services_map] list of index nodes in cluster: [ip:10.112.205.101 port:8091 ssh_username:root, ip:10.112.205.102 port:8091 ssh_username:root, ip:10.112.205.103 port:8091 ssh_username:root] 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [rest_client.set_index_settings_internal] {'indexer.settings.storage_mode': 'plasma'} set 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [newtuq.setUp] Allowing the indexer to complete restart after setting the internal settings 2020-11-10 13:01:11 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:01:16 | INFO | MainProcess | MainThread | [rest_client.set_index_settings_internal] {'indexer.api.enableTestServer': True} set 2020-11-10 13:01:16 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:01:16 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:01:16 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:01:16 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:01:25 | INFO | MainProcess | MainThread | [basetestcase.load] LOAD IS FINISHED 2020-11-10 13:01:25 | INFO | MainProcess | MainThread | [basetestcase.get_nodes_from_services_map] list of n1ql nodes in cluster: [ip:10.112.205.101 port:8091 ssh_username:root] 2020-11-10 13:01:25 | INFO | MainProcess | MainThread | [newtuq.setUp] ip:10.112.205.101 port:8091 ssh_username:root 2020-11-10 13:01:25 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 30 secs. ... 2020-11-10 13:01:55 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:01:56 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:01:56 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:01:56 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:01:56 | INFO | MainProcess | MainThread | [collections_concurrent_indexes.setUp] ============== ConcurrentIndexes setup has started ============== 2020-11-10 13:01:56 | INFO | MainProcess | Cluster_Thread | [rest_client.create_bucket] http://10.112.205.101:8091/pools/default/buckets with param: name=test_bucket&ramQuotaMB=100&authType=none&replicaNumber=1&bucketType=membase&replicaIndex=1&threadsNumber=3&flushEnabled=1&evictionPolicy=valueOnly&compressionMode=passive&storageBackend=couchstore 2020-11-10 13:01:56 | INFO | MainProcess | Cluster_Thread | [rest_client.create_bucket] 0.01 seconds to create bucket test_bucket 2020-11-10 13:01:56 | INFO | MainProcess | Cluster_Thread | [bucket_helper.wait_for_memcached] waiting for memcached bucket : test_bucket in 10.112.205.101 to accept set ops 2020-11-10 13:01:58 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:01:58 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:01:58 | WARNING | MainProcess | Cluster_Thread | [task.check] vbucket map not ready after try 0 2020-11-10 13:01:58 | INFO | MainProcess | Cluster_Thread | [bucket_helper.wait_for_memcached] waiting for memcached bucket : test_bucket in 10.112.205.101 to accept set ops 2020-11-10 13:01:58 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:01:59 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:01:59 | WARNING | MainProcess | Cluster_Thread | [task.check] vbucket map not ready after try 1 2020-11-10 13:01:59 | INFO | MainProcess | Cluster_Thread | [bucket_helper.wait_for_memcached] waiting for memcached bucket : test_bucket in 10.112.205.101 to accept set ops 2020-11-10 13:01:59 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:01:59 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:01:59 | WARNING | MainProcess | Cluster_Thread | [task.check] vbucket map not ready after try 2 2020-11-10 13:01:59 | INFO | MainProcess | Cluster_Thread | [bucket_helper.wait_for_memcached] waiting for memcached bucket : test_bucket in 10.112.205.101 to accept set ops 2020-11-10 13:01:59 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:02:00 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:02:00 | WARNING | MainProcess | Cluster_Thread | [task.check] vbucket map not ready after try 3 2020-11-10 13:02:00 | INFO | MainProcess | Cluster_Thread | [bucket_helper.wait_for_memcached] waiting for memcached bucket : test_bucket in 10.112.205.101 to accept set ops 2020-11-10 13:02:00 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:02:00 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:02:00 | WARNING | MainProcess | Cluster_Thread | [task.check] vbucket map not ready after try 4 2020-11-10 13:02:00 | INFO | MainProcess | Cluster_Thread | [bucket_helper.wait_for_memcached] waiting for memcached bucket : test_bucket in 10.112.205.101 to accept set ops 2020-11-10 13:02:00 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:02:00 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:02:01 | WARNING | MainProcess | Cluster_Thread | [task.check] vbucket map not ready after try 5 2020-11-10 13:02:01 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:02:01 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:02:01 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:02:01 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:02:01 | INFO | MainProcess | MainThread | [collections_concurrent_indexes.setUp] ============== ConcurrentIndexes setup has completed ============== 2020-11-10 13:02:01 | INFO | MainProcess | MainThread | [collections_concurrent_indexes.tearDown] ============== ConcurrentIndexes tearDown has started ============== 2020-11-10 13:02:01 | INFO | MainProcess | MainThread | [basetestcase.get_nodes_from_services_map] list of index nodes in cluster: [ip:10.112.205.101 port:8091 ssh_username:root, ip:10.112.205.102 port:8091 ssh_username:root, ip:10.112.205.103 port:8091 ssh_username:root] 2020-11-10 13:02:01 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:02:01 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:02:01 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:02:01 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:02:01 | INFO | MainProcess | MainThread | [rest_client.diag_eval] /diag/eval status on 10.112.205.101:8091: True content: "/opt/couchbase/var/lib/couchbase/logs" command: filename:absname(element(2, application:get_env(ns_server,error_logger_mf_dir))). 2020-11-10 13:02:01 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: zgrep "panic" "/opt/couchbase/var/lib/couchbase/logs"/indexer.log* | wc -l 2020-11-10 13:02:02 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:02:02 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:02:02 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:02:02 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:02:02 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:02:02 | INFO | MainProcess | MainThread | [rest_client.diag_eval] /diag/eval status on 10.112.205.103:8091: True content: "/opt/couchbase/var/lib/couchbase/logs" command: filename:absname(element(2, application:get_env(ns_server,error_logger_mf_dir))). 2020-11-10 13:02:02 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: zgrep "panic" "/opt/couchbase/var/lib/couchbase/logs"/indexer.log* | wc -l 2020-11-10 13:02:02 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:02:02 | INFO | MainProcess | MainThread | [basetestcase.get_nodes_from_services_map] list of kv nodes in cluster: [ip:10.112.205.101 port:8091 ssh_username:root] 2020-11-10 13:02:02 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:02:03 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:02:03 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:02:03 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:02:03 | INFO | MainProcess | MainThread | [rest_client.diag_eval] /diag/eval status on 10.112.205.101:8091: True content: "/opt/couchbase/var/lib/couchbase/logs" command: filename:absname(element(2, application:get_env(ns_server,error_logger_mf_dir))). 2020-11-10 13:02:03 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: zgrep "panic" "/opt/couchbase/var/lib/couchbase/logs"/projector.log* | wc -l 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [basetestcase.get_nodes_from_services_map] list of n1ql nodes in cluster: [ip:10.112.205.101 port:8091 ssh_username:root] 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [tuq_helper.drop_primary_index] CHECK FOR PRIMARY INDEXES 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [basetestcase.print_cluster_stats] ------- Cluster statistics ------- 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [basetestcase.print_cluster_stats] 10.112.205.101:8091 => {'services': ['index', 'kv', 'n1ql'], 'cpu_utilization': 42.05069124423963, 'mem_free': 2339602432, 'mem_total': 2962632704, 'swap_mem_used': 0, 'swap_mem_total': 1107292160} 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [basetestcase.print_cluster_stats] 10.112.205.102:8091 => {'services': ['index'], 'cpu_utilization': 1.740020470829069, 'mem_free': 1482502144, 'mem_total': 1930829824, 'swap_mem_used': 0, 'swap_mem_total': 1107292160} 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [basetestcase.print_cluster_stats] 10.112.205.103:8091 => {'services': ['index'], 'cpu_utilization': 5.202913631633715, 'mem_free': 1465286656, 'mem_total': 1930829824, 'swap_mem_used': 0, 'swap_mem_total': 1107292160} 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [basetestcase.print_cluster_stats] --- End of cluster statistics --- 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [basetestcase.tearDown] ============== basetestcase cleanup was started for test #1 suite_setUp ============== 2020-11-10 13:02:05 | WARNING | MainProcess | MainThread | [basetestcase.tearDown] Alerts were found: [{'msg': "The time on node 'ns_1@10.112.205.102' is not synchronized. Please ensure that NTP is set up correctly on all nodes and that clocks are synchronized.", 'serverTime': '2020-11-10T07:31:57.000Z'}] 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [bucket_helper.delete_all_buckets_or_assert] deleting existing buckets ['test_bucket'] on 10.112.205.101 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [bucket_helper.wait_for_bucket_deletion] waiting for bucket deletion to complete.... 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [rest_client.bucket_exists] node 10.112.205.101 existing buckets : [] 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [bucket_helper.delete_all_buckets_or_assert] deleted bucket : test_bucket from 10.112.205.101 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.102, nothing to delete 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.103, nothing to delete 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.104, nothing to delete 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [cluster_helper.cleanup_cluster] rebalancing all nodes in order to remove nodes 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [rest_client.rebalance] rebalance params : {'knownNodes': 'ns_1@10.112.205.101,ns_1@10.112.205.102,ns_1@10.112.205.103', 'ejectedNodes': 'ns_1@10.112.205.102,ns_1@10.112.205.103', 'user': 'Administrator', 'password': 'password'} 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [rest_client.rebalance] rebalance operation started 2020-11-10 13:02:05 | INFO | MainProcess | MainThread | [rest_client._rebalance_status_and_progress] rebalance percentage : 0.00 % 2020-11-10 13:02:25 | INFO | MainProcess | MainThread | [rest_client.monitorRebalance] rebalance progress took 20.02 seconds 2020-11-10 13:02:25 | INFO | MainProcess | MainThread | [rest_client.monitorRebalance] sleep for 10 seconds after rebalance... 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [cluster_helper.cleanup_cluster] removed all the nodes from cluster associated with ip:10.112.205.101 port:8091 ssh_username:root ? [('ns_1@10.112.205.102', 8091), ('ns_1@10.112.205.103', 8091)] 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.101:8091 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.101:8091 is running 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.102:8091 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.102:8091 is running 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.103:8091 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.103:8091 is running 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.104:8091 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.104:8091 is running 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [basetestcase.tearDown] ============== basetestcase cleanup was finished for test #1 suite_setUp ============== 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [basetestcase.tearDown] closing all ssh connections 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [basetestcase.tearDown] closing all memcached connections 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:02:35 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:02:36 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:02:36 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:02:36 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:02:36 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:02:36 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:02:36 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.101 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:02:36 | INFO | MainProcess | MainThread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:02:36 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl stop couchbase-server.service 2020-11-10 13:02:38 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:02:38 | INFO | MainProcess | MainThread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:02:38 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:02:43 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:02:43 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:02:43 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:02:43 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:02:43 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:02:43 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:02:43 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:02:44 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:02:44 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.101 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:02:44 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:02:44 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:02:44 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:02:44 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl start couchbase-server.service 2020-11-10 13:02:44 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:02:44 | INFO | MainProcess | MainThread | [remote_util.sleep] 10.112.205.101:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:02:49 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:02:49 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:02:49 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Couchbase server status: [' Process: 3648 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:02:49 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:02:49 | INFO | MainProcess | MainThread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.101: with pid 3729 2020-11-10 13:02:49 | INFO | MainProcess | MainThread | [basetestcase.start_server] Couchbase started 2020-11-10 13:02:49 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 13:02:49 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 13:02:49 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:02:49 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:02:49 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 13:02:49 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 13:02:49 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:02:50 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:02:50 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.102 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:02:50 | INFO | MainProcess | MainThread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:02:50 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl stop couchbase-server.service 2020-11-10 13:02:52 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:02:52 | INFO | MainProcess | MainThread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:02:52 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:02:57 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:02:57 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:02:57 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:02:57 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:02:57 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 13:02:57 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 13:02:57 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:02:57 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:02:58 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.102 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:02:58 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:02:58 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:02:58 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:02:58 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl start couchbase-server.service 2020-11-10 13:02:58 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:02:58 | INFO | MainProcess | MainThread | [remote_util.sleep] 10.112.205.102:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:03:03 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:03:03 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:03 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Couchbase server status: [' Process: 3494 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:03:03 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:03:03 | INFO | MainProcess | MainThread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.102: with pid 3575 2020-11-10 13:03:03 | INFO | MainProcess | MainThread | [basetestcase.start_server] Couchbase started 2020-11-10 13:03:03 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:03:03 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:03:03 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:03:03 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:03:03 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:03:03 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:03:03 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:03:04 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:03:04 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.103 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:03:04 | INFO | MainProcess | MainThread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:03:04 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl stop couchbase-server.service 2020-11-10 13:03:06 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:06 | INFO | MainProcess | MainThread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:03:06 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.103 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl start couchbase-server.service 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:11 | INFO | MainProcess | MainThread | [remote_util.sleep] 10.112.205.103:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:03:16 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:03:16 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:16 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Couchbase server status: [' Process: 3519 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:03:16 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:03:16 | INFO | MainProcess | MainThread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.103: with pid 3600 2020-11-10 13:03:16 | INFO | MainProcess | MainThread | [basetestcase.start_server] Couchbase started 2020-11-10 13:03:16 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:03:16 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:03:16 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:03:17 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:03:17 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:03:17 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:03:17 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:03:17 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:03:17 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.104 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:03:17 | INFO | MainProcess | MainThread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:03:17 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl stop couchbase-server.service 2020-11-10 13:03:19 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:19 | INFO | MainProcess | MainThread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:03:19 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:03:24 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:03:24 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:24 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:03:24 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:24 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:03:24 | INFO | MainProcess | MainThread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:03:24 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:03:24 | INFO | MainProcess | MainThread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:03:24 | INFO | MainProcess | MainThread | [remote_util.is_couchbase_installed] 10.112.205.104 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:03:24 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:03:25 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:03:25 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:03:25 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl start couchbase-server.service 2020-11-10 13:03:25 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:25 | INFO | MainProcess | MainThread | [remote_util.sleep] 10.112.205.104:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:03:30 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:03:30 | INFO | MainProcess | MainThread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:30 | INFO | MainProcess | MainThread | [remote_util.start_couchbase] Couchbase server status: [' Process: 3219 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:03:30 | INFO | MainProcess | MainThread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:03:30 | INFO | MainProcess | MainThread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.104: with pid 3300 2020-11-10 13:03:30 | INFO | MainProcess | MainThread | [basetestcase.start_server] Couchbase started 2020-11-10 13:03:30 | INFO | MainProcess | MainThread | [basetestcase.sleep] sleep for 10 secs. ... Cluster instance shutdown with force 2020-11-10 13:03:40 | INFO | MainProcess | MainThread | [collections_concurrent_indexes.tearDown] ============== ConcurrentIndexes tearDown has completed ============== ok ---------------------------------------------------------------------- Ran 1 test in 303.294s OK test_index_creation_with_rebalanced_out_assigned_index_node (gsi.collections_concurrent_indexes.ConcurrentIndexes) https://issues.couchbase.com/browse/MB-41897 ... -->result: 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [rest_client.get_nodes_version] Node version in cluster 7.0.0-3649-enterprise 2020-11-10 13:03:40 | ERROR | MainProcess | test_thread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [rest_client.get_nodes_versions] Node versions in cluster [] 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [basetestcase.setUp] ============== basetestcase setup was started for test #1 test_index_creation_with_rebalanced_out_assigned_index_node============== 2020-11-10 13:03:40 | ERROR | MainProcess | test_thread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default/ body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.101, nothing to delete 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.102, nothing to delete 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.103, nothing to delete 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.104, nothing to delete 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.101:8091 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.101:8091 is running 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.102:8091 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.102:8091 is running 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.103:8091 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.103:8091 is running 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.104:8091 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.104:8091 is running 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [collections_concurrent_indexes.tearDown] ============== ConcurrentIndexes tearDown has started ============== 2020-11-10 13:03:40 | ERROR | MainProcess | test_thread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:03:40 | ERROR | MainProcess | test_thread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [basetestcase.get_nodes_from_services_map] cannot find service node index in cluster 2020-11-10 13:03:40 | ERROR | MainProcess | test_thread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [basetestcase.print_cluster_stats] ------- Cluster statistics ------- 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [basetestcase.print_cluster_stats] --- End of cluster statistics --- 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [basetestcase.tearDown] ============== basetestcase cleanup was started for test #1 test_index_creation_with_rebalanced_out_assigned_index_node ============== 2020-11-10 13:03:40 | ERROR | MainProcess | test_thread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default/ body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.101, nothing to delete 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.102, nothing to delete 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.103, nothing to delete 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [bucket_helper.delete_all_buckets_or_assert] Could not find any buckets for node 10.112.205.104, nothing to delete 2020-11-10 13:03:40 | INFO | MainProcess | test_thread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.101:8091 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.101:8091 is running 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.102:8091 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.102:8091 is running 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.103:8091 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.103:8091 is running 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] waiting for ns_server @ 10.112.205.104:8091 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [rest_client.is_ns_server_running] -->is_ns_server_running? 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [cluster_helper.wait_for_ns_servers_or_assert] ns_server @ 10.112.205.104:8091 is running 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [basetestcase.tearDown] ============== basetestcase cleanup was finished for test #1 test_index_creation_with_rebalanced_out_assigned_index_node ============== 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [basetestcase.tearDown] closing all ssh connections 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [basetestcase.tearDown] closing all memcached connections 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:03:41 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.101 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:03:42 | INFO | MainProcess | test_thread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:03:42 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl stop couchbase-server.service 2020-11-10 13:03:43 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:43 | INFO | MainProcess | test_thread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:03:43 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:03:48 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:03:48 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:48 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:03:48 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:48 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:03:48 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:03:49 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:03:49 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:03:49 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.101 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:03:49 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:03:49 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:03:49 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:03:49 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl start couchbase-server.service 2020-11-10 13:03:49 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:49 | INFO | MainProcess | test_thread | [remote_util.sleep] 10.112.205.101:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:03:54 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:03:54 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:54 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Couchbase server status: [' Process: 4078 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:03:54 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:03:54 | INFO | MainProcess | test_thread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.101: with pid 4159 2020-11-10 13:03:54 | INFO | MainProcess | test_thread | [basetestcase.start_server] Couchbase started 2020-11-10 13:03:54 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 13:03:54 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 13:03:54 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:03:54 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:03:54 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 13:03:55 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 13:03:55 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:03:55 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:03:55 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.102 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:03:55 | INFO | MainProcess | test_thread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:03:55 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl stop couchbase-server.service 2020-11-10 13:03:57 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:03:57 | INFO | MainProcess | test_thread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:03:57 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.102 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl start couchbase-server.service 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:02 | INFO | MainProcess | test_thread | [remote_util.sleep] 10.112.205.102:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:04:07 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:04:07 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:07 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Couchbase server status: [' Process: 3897 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:04:07 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:04:08 | INFO | MainProcess | test_thread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.102: with pid 3978 2020-11-10 13:04:08 | INFO | MainProcess | test_thread | [basetestcase.start_server] Couchbase started 2020-11-10 13:04:08 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:04:08 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:04:08 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:04:08 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:04:08 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:04:08 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:04:08 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:04:08 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:04:08 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.103 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:04:08 | INFO | MainProcess | test_thread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:04:08 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl stop couchbase-server.service 2020-11-10 13:04:10 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:10 | INFO | MainProcess | test_thread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:04:10 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:04:15 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:04:15 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:15 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:04:15 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:15 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:04:15 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:04:15 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:04:15 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:04:16 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.103 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:04:16 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:04:16 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:04:16 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:04:16 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl start couchbase-server.service 2020-11-10 13:04:16 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:16 | INFO | MainProcess | test_thread | [remote_util.sleep] 10.112.205.103:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:04:21 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:04:21 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:21 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Couchbase server status: [' Process: 3921 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:04:21 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:04:21 | INFO | MainProcess | test_thread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.103: with pid 4002 2020-11-10 13:04:21 | INFO | MainProcess | test_thread | [basetestcase.start_server] Couchbase started 2020-11-10 13:04:21 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:04:21 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:04:21 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:04:21 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:04:21 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:04:21 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:04:21 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:04:22 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:04:22 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.104 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:04:22 | INFO | MainProcess | test_thread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:04:22 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl stop couchbase-server.service 2020-11-10 13:04:23 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:23 | INFO | MainProcess | test_thread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:04:23 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:04:28 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:04:28 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:28 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:04:28 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:28 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:04:29 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:04:29 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:04:29 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:04:29 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.104 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:04:29 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:04:29 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:04:29 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:04:29 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl start couchbase-server.service 2020-11-10 13:04:29 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:29 | INFO | MainProcess | test_thread | [remote_util.sleep] 10.112.205.104:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:04:34 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:04:34 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:34 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Couchbase server status: [' Process: 3621 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:04:34 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:04:34 | INFO | MainProcess | test_thread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.104: with pid 3702 2020-11-10 13:04:34 | INFO | MainProcess | test_thread | [basetestcase.start_server] Couchbase started 2020-11-10 13:04:34 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 10 secs. ... Cluster instance shutdown with force 2020-11-10 13:04:44 | INFO | MainProcess | test_thread | [collections_concurrent_indexes.tearDown] ============== ConcurrentIndexes tearDown has completed ============== 2020-11-10 13:04:44 | INFO | MainProcess | test_thread | [basetestcase.setUp] initializing cluster 2020-11-10 13:04:44 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:04:44 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:04:44 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:04:45 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:04:45 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:04:45 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:04:45 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:04:45 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:04:45 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.101 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:04:45 | INFO | MainProcess | test_thread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:04:45 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl stop couchbase-server.service 2020-11-10 13:04:47 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:47 | INFO | MainProcess | test_thread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:04:47 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:04:52 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:04:52 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:52 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:04:52 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:52 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:04:52 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:04:52 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:04:52 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:04:52 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.101 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:04:52 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:04:53 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:04:53 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:04:53 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl start couchbase-server.service 2020-11-10 13:04:53 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:53 | INFO | MainProcess | test_thread | [remote_util.sleep] 10.112.205.101:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:04:58 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:04:58 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:04:58 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Couchbase server status: [' Process: 4479 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:04:58 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:04:58 | INFO | MainProcess | test_thread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.101: with pid 4560 2020-11-10 13:04:58 | INFO | MainProcess | test_thread | [basetestcase.start_server] Couchbase started 2020-11-10 13:04:58 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 13:04:58 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 13:04:58 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:04:58 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:04:58 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 13:04:58 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 13:04:58 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:04:59 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:04:59 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.102 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:04:59 | INFO | MainProcess | test_thread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:04:59 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl stop couchbase-server.service 2020-11-10 13:05:01 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:01 | INFO | MainProcess | test_thread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:05:01 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.102 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl start couchbase-server.service 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:06 | INFO | MainProcess | test_thread | [remote_util.sleep] 10.112.205.102:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:05:11 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:05:11 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:11 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Couchbase server status: [' Process: 4298 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:05:11 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:05:11 | INFO | MainProcess | test_thread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.102: with pid 4379 2020-11-10 13:05:11 | INFO | MainProcess | test_thread | [basetestcase.start_server] Couchbase started 2020-11-10 13:05:11 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:05:11 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:05:11 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:05:12 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:05:12 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:05:12 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:05:12 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:05:12 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:05:12 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.103 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:05:12 | INFO | MainProcess | test_thread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:05:12 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl stop couchbase-server.service 2020-11-10 13:05:14 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:14 | INFO | MainProcess | test_thread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:05:14 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:05:19 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:05:19 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:19 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:05:19 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:19 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:05:19 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:05:19 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:05:20 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:05:20 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.103 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:05:20 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:05:20 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:05:20 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:05:20 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl start couchbase-server.service 2020-11-10 13:05:20 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:20 | INFO | MainProcess | test_thread | [remote_util.sleep] 10.112.205.103:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:05:25 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:05:25 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:25 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Couchbase server status: [' Process: 4322 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:05:25 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:05:25 | INFO | MainProcess | test_thread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.103: with pid 4403 2020-11-10 13:05:25 | INFO | MainProcess | test_thread | [basetestcase.start_server] Couchbase started 2020-11-10 13:05:25 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:05:25 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:05:25 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:05:25 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:05:25 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:05:25 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:05:25 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:05:26 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:05:26 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.104 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:05:26 | INFO | MainProcess | test_thread | [remote_util.stop_couchbase] Running systemd command on this server 2020-11-10 13:05:26 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl stop couchbase-server.service 2020-11-10 13:05:28 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:28 | INFO | MainProcess | test_thread | [basetestcase.stop_server] Couchbase stopped 2020-11-10 13:05:28 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.is_couchbase_installed] 10.112.205.104 **** The linux version file /opt/couchbase/ VERSION.txt exists 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Starting couchbase server 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Running systemd command on this server 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl start couchbase-server.service 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:33 | INFO | MainProcess | test_thread | [remote_util.sleep] 10.112.205.104:sleep for 5 secs. waiting for couchbase server to come up ... 2020-11-10 13:05:38 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2020-11-10 13:05:38 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed successfully with root 2020-11-10 13:05:38 | INFO | MainProcess | test_thread | [remote_util.start_couchbase] Couchbase server status: [' Process: 4022 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2020-11-10 13:05:38 | INFO | MainProcess | test_thread | [remote_util.is_process_running] Checking for process beam.smp on linux 2020-11-10 13:05:38 | INFO | MainProcess | test_thread | [remote_util.is_process_running] process beam.smp is running on 10.112.205.104: with pid 4103 2020-11-10 13:05:38 | INFO | MainProcess | test_thread | [basetestcase.start_server] Couchbase started 2020-11-10 13:05:38 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 10 secs. ... 2020-11-10 13:05:48 | INFO | MainProcess | Cluster_Thread | [task.execute] server: ip:10.112.205.101 port:8091 ssh_username:root, nodes/self 2020-11-10 13:05:48 | INFO | MainProcess | Cluster_Thread | [task.execute] {'uptime': '49', 'memoryTotal': 2962632704, 'memoryFree': 2568208384, 'mcdMemoryReserved': 2260, 'mcdMemoryAllocated': 2260, 'status': 'healthy', 'hostname': '10.112.205.101:8091', 'clusterCompatibility': 458752, 'clusterMembership': 'active', 'version': '7.0.0-3649-enterprise', 'os': 'x86_64-unknown-linux-gnu', 'ports': [], 'availableStorage': [], 'storage': [], 'memoryQuota': 980, 'moxi': 11211, 'memcached': 11210, 'id': 'ns_1@10.112.205.101', 'ip': '10.112.205.101', 'rest_username': '', 'rest_password': '', 'port': '8091', 'services': ['kv'], 'storageTotalRam': 2825, 'curr_items': 0} 2020-11-10 13:05:48 | ERROR | MainProcess | Cluster_Thread | [rest_client._http_request] GET http://10.112.205.101:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:05:48 | INFO | MainProcess | Cluster_Thread | [task.execute] quota for index service will be 256 MB 2020-11-10 13:05:48 | INFO | MainProcess | Cluster_Thread | [task.execute] set index quota to node 10.112.205.101 2020-11-10 13:05:48 | INFO | MainProcess | Cluster_Thread | [rest_client.set_service_memoryQuota] pools/default params : indexMemoryQuota=256 2020-11-10 13:05:48 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster_memoryQuota] pools/default params : memoryQuota=1258 2020-11-10 13:05:48 | INFO | MainProcess | Cluster_Thread | [rest_client.init_node_services] --> init_node_services(Administrator,password,10.112.205.101,8091,['kv', 'n1ql', 'index']) 2020-11-10 13:05:48 | INFO | MainProcess | Cluster_Thread | [rest_client.init_node_services] /node/controller/setupServices params on 10.112.205.101: 8091:hostname=10.112.205.101&user=Administrator&password=password&services=kv%2Cn1ql%2Cindex 2020-11-10 13:05:48 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> in init_cluster...Administrator,password,8091 2020-11-10 13:05:48 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] settings/web params on 10.112.205.101:8091:port=8091&username=Administrator&password=password 2020-11-10 13:05:49 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> status:True 2020-11-10 13:05:49 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:05:49 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:05:49 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:05:49 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:05:49 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2020-11-10 13:05:49 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] command executed with root but got an error [' % Total % Received % Xferd Average Speed Time Time Time Current', ' Dload Upload Total Spent Left Speed', '', ' 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0', '100 43 100 2 100 41 34 710 --:--:-- --:--:-- --:--:-- 719'] ... 2020-11-10 13:05:49 | INFO | MainProcess | Cluster_Thread | [remote_util.enable_diag_eval_on_non_local_hosts] ['ok'] 2020-11-10 13:05:49 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.101:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:05:49 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.101:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:05:49 | INFO | MainProcess | Cluster_Thread | [rest_client.set_indexer_storage_mode] settings/indexes params : storageMode=plasma 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [task.execute] server: ip:10.112.205.102 port:8091 ssh_username:root, nodes/self 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [task.execute] {'uptime': '38', 'memoryTotal': 1930829824, 'memoryFree': 1574412288, 'mcdMemoryReserved': 1473, 'mcdMemoryAllocated': 1473, 'status': 'healthy', 'hostname': '10.112.205.102:8091', 'clusterCompatibility': 458752, 'clusterMembership': 'active', 'version': '7.0.0-3649-enterprise', 'os': 'x86_64-unknown-linux-gnu', 'ports': [], 'availableStorage': [], 'storage': [], 'memoryQuota': 256, 'moxi': 11211, 'memcached': 11210, 'id': 'ns_1@10.112.205.102', 'ip': '10.112.205.102', 'rest_username': '', 'rest_password': '', 'port': '8091', 'services': ['kv'], 'storageTotalRam': 1841, 'curr_items': 0} 2020-11-10 13:05:50 | ERROR | MainProcess | Cluster_Thread | [rest_client._http_request] GET http://10.112.205.102:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster_memoryQuota] pools/default params : memoryQuota=986 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> in init_cluster...Administrator,password,8091 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] settings/web params on 10.112.205.102:8091:port=8091&username=Administrator&password=password 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> status:True 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.102 as root 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.102: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] command executed with root but got an error [' % Total % Received % Xferd Average Speed Time Time Time Current', ' Dload Upload Total Spent Left Speed', '', ' 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0', '100 43 100 2 100 41 13 266 --:--:-- --:--:-- --:--:-- 267'] ... 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [remote_util.enable_diag_eval_on_non_local_hosts] ['ok'] 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.102:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.102:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:05:50 | INFO | MainProcess | Cluster_Thread | [rest_client.set_indexer_storage_mode] settings/indexes params : storageMode=plasma 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [task.execute] server: ip:10.112.205.103 port:8091 ssh_username:root, nodes/self 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [task.execute] {'uptime': '28', 'memoryTotal': 1930829824, 'memoryFree': 1572179968, 'mcdMemoryReserved': 1473, 'mcdMemoryAllocated': 1473, 'status': 'healthy', 'hostname': '10.112.205.103:8091', 'clusterCompatibility': 458752, 'clusterMembership': 'active', 'version': '7.0.0-3649-enterprise', 'os': 'x86_64-unknown-linux-gnu', 'ports': [], 'availableStorage': [], 'storage': [], 'memoryQuota': 256, 'moxi': 11211, 'memcached': 11210, 'id': 'ns_1@10.112.205.103', 'ip': '10.112.205.103', 'rest_username': '', 'rest_password': '', 'port': '8091', 'services': ['kv'], 'storageTotalRam': 1841, 'curr_items': 0} 2020-11-10 13:05:51 | ERROR | MainProcess | Cluster_Thread | [rest_client._http_request] GET http://10.112.205.103:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster_memoryQuota] pools/default params : memoryQuota=986 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> in init_cluster...Administrator,password,8091 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] settings/web params on 10.112.205.103:8091:port=8091&username=Administrator&password=password 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> status:True 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.103 as root 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.103: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] command executed with root but got an error [' % Total % Received % Xferd Average Speed Time Time Time Current', ' Dload Upload Total Spent Left Speed', '', ' 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0', '100 43 100 2 100 41 13 266 --:--:-- --:--:-- --:--:-- 267'] ... 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [remote_util.enable_diag_eval_on_non_local_hosts] ['ok'] 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.103:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.103:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [rest_client.set_indexer_storage_mode] settings/indexes params : storageMode=plasma 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [task.execute] server: ip:10.112.205.104 port:8091 ssh_username:root, nodes/self 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [task.execute] {'uptime': '13', 'memoryTotal': 1930829824, 'memoryFree': 1570054144, 'mcdMemoryReserved': 1473, 'mcdMemoryAllocated': 1473, 'status': 'healthy', 'hostname': '10.112.205.104:8091', 'clusterCompatibility': 458752, 'clusterMembership': 'active', 'version': '7.0.0-3649-enterprise', 'os': 'x86_64-unknown-linux-gnu', 'ports': [], 'availableStorage': [], 'storage': [], 'memoryQuota': 256, 'moxi': 11211, 'memcached': 11210, 'id': 'ns_1@10.112.205.104', 'ip': '10.112.205.104', 'rest_username': '', 'rest_password': '', 'port': '8091', 'services': ['kv'], 'storageTotalRam': 1841, 'curr_items': 0} 2020-11-10 13:05:51 | ERROR | MainProcess | Cluster_Thread | [rest_client._http_request] GET http://10.112.205.104:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster_memoryQuota] pools/default params : memoryQuota=986 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> in init_cluster...Administrator,password,8091 2020-11-10 13:05:51 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] settings/web params on 10.112.205.104:8091:port=8091&username=Administrator&password=password 2020-11-10 13:05:52 | INFO | MainProcess | Cluster_Thread | [rest_client.init_cluster] --> status:True 2020-11-10 13:05:52 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2020-11-10 13:05:52 | INFO | MainProcess | Cluster_Thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.104 as root 2020-11-10 13:05:52 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:05:52 | INFO | MainProcess | Cluster_Thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:05:52 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.104: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2020-11-10 13:05:52 | INFO | MainProcess | Cluster_Thread | [remote_util.execute_command_raw] command executed with root but got an error [' % Total % Received % Xferd Average Speed Time Time Time Current', ' Dload Upload Total Spent Left Speed', '', ' 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0', '100 43 100 2 100 41 12 265 --:--:-- --:--:-- --:--:-- 266'] ... 2020-11-10 13:05:52 | INFO | MainProcess | Cluster_Thread | [remote_util.enable_diag_eval_on_non_local_hosts] ['ok'] 2020-11-10 13:05:52 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.104:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:05:52 | INFO | MainProcess | Cluster_Thread | [rest_client.diag_eval] /diag/eval status on 10.112.205.104:8091: True content: [7,0] command: cluster_compat_mode:get_compat_version(). 2020-11-10 13:05:52 | INFO | MainProcess | Cluster_Thread | [rest_client.set_indexer_storage_mode] settings/indexes params : storageMode=plasma 2020-11-10 13:05:52 | INFO | MainProcess | test_thread | [basetestcase.add_built_in_server_user] **** add built-in 'cbadminbucket' user to node 10.112.205.101 **** 2020-11-10 13:05:52 | ERROR | MainProcess | test_thread | [rest_client._http_request] DELETE http://10.112.205.101:8091/settings/rbac/users/local/cbadminbucket body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"User was not found."' auth: Administrator:password 2020-11-10 13:05:52 | INFO | MainProcess | test_thread | [internal_user.delete_user] Exception while deleting user. Exception is -b'"User was not found."' 2020-11-10 13:05:52 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:05:57 | INFO | MainProcess | test_thread | [basetestcase.add_built_in_server_user] **** add 'admin' role to 'cbadminbucket' user **** 2020-11-10 13:05:57 | INFO | MainProcess | test_thread | [basetestcase.setUp] done initializing cluster 2020-11-10 13:05:58 | INFO | MainProcess | Cluster_Thread | [task.add_nodes] adding node 10.112.205.102:8091 to cluster 2020-11-10 13:05:58 | INFO | MainProcess | Cluster_Thread | [rest_client.add_node] adding remote node @10.112.205.102:8091 to this cluster @10.112.205.101:8091 2020-11-10 13:06:00 | INFO | MainProcess | Cluster_Thread | [task.add_nodes] adding node 10.112.205.103:8091 to cluster 2020-11-10 13:06:00 | INFO | MainProcess | Cluster_Thread | [rest_client.add_node] adding remote node @10.112.205.103:8091 to this cluster @10.112.205.101:8091 2020-11-10 13:06:02 | INFO | MainProcess | Cluster_Thread | [rest_client.rebalance] rebalance params : {'knownNodes': 'ns_1@10.112.205.101,ns_1@10.112.205.102,ns_1@10.112.205.103', 'ejectedNodes': '', 'user': 'Administrator', 'password': 'password'} 2020-11-10 13:06:02 | INFO | MainProcess | Cluster_Thread | [rest_client.rebalance] rebalance operation started 2020-11-10 13:06:02 | INFO | MainProcess | Cluster_Thread | [rest_client._rebalance_status_and_progress] rebalance percentage : 0.00 % 2020-11-10 13:06:02 | INFO | MainProcess | Cluster_Thread | [task.check] Rebalance - status: running, progress: 0.00% 2020-11-10 13:06:12 | INFO | MainProcess | Cluster_Thread | [task.check] Rebalance - status: none, progress: 100.00% 2020-11-10 13:06:12 | INFO | MainProcess | Cluster_Thread | [task.check] rebalancing was completed with progress: 100% in 10.044234037399292 sec 2020-11-10 13:06:12 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:06:12 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] running command.raw on 10.112.205.101: curl http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [remote_util.execute_command_raw] command executed with root but got an error [' % Total % Received % Xferd Average Speed Time Time Time Current', ' Dload Upload Total Spent Left Speed', '', ' 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0', '100 43 100 2 100 41 12 265 --:--:-- --:--:-- --:--:-- 266'] ... 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [remote_util.enable_diag_eval_on_non_local_hosts] ['ok'] 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [basetestcase.enable_diag_eval_on_non_local_hosts] Enabled diag/eval for non-local hosts from 10.112.205.101 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [basetestcase.setUp] ============== basetestcase setup was finished for test #1 test_index_creation_with_rebalanced_out_assigned_index_node ============== 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [basetestcase.print_cluster_stats] ------- Cluster statistics ------- 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [basetestcase.print_cluster_stats] 10.112.205.101:8091 => {'services': ['index', 'kv', 'n1ql'], 'cpu_utilization': 7.749469214437367, 'mem_free': 2460483584, 'mem_total': 2962632704, 'swap_mem_used': 0, 'swap_mem_total': 1107292160} 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [basetestcase.print_cluster_stats] 10.112.205.102:8091 => {'services': ['index'], 'cpu_utilization': 5.785123966942149, 'mem_free': 1556398080, 'mem_total': 1930829824, 'swap_mem_used': 0, 'swap_mem_total': 1107292160} 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [basetestcase.print_cluster_stats] 10.112.205.103:8091 => {'services': ['index'], 'cpu_utilization': 5.658436213991769, 'mem_free': 1559851008, 'mem_total': 1930829824, 'swap_mem_used': 0, 'swap_mem_total': 1107292160} 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [basetestcase.print_cluster_stats] --- End of cluster statistics --- 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [newtuq.setUp] Initial status of 10.112.205.101 cluster is healthy 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [newtuq.setUp] current status of 10.112.205.101 is healthy 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [newtuq.setUp] Initial status of 10.112.205.102 cluster is healthy 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [newtuq.setUp] current status of 10.112.205.102 is healthy 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [newtuq.setUp] Initial status of 10.112.205.103 cluster is healthy 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [newtuq.setUp] current status of 10.112.205.103 is healthy 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [newtuq.setUp] Initial status of 10.112.205.104 cluster is healthy 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [newtuq.setUp] current status of 10.112.205.104 is healthy 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [basetestcase.get_nodes_from_services_map] list of index nodes in cluster: [ip:10.112.205.101 port:8091 ssh_username:root, ip:10.112.205.102 port:8091 ssh_username:root, ip:10.112.205.103 port:8091 ssh_username:root] 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [rest_client.set_index_settings_internal] {'indexer.settings.storage_mode': 'plasma'} set 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [newtuq.setUp] Allowing the indexer to complete restart after setting the internal settings 2020-11-10 13:06:13 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:06:18 | INFO | MainProcess | test_thread | [rest_client.set_index_settings_internal] {'indexer.api.enableTestServer': True} set 2020-11-10 13:06:18 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:06:18 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:06:18 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:06:19 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:06:28 | INFO | MainProcess | test_thread | [basetestcase.load] LOAD IS FINISHED 2020-11-10 13:06:28 | INFO | MainProcess | test_thread | [basetestcase.get_nodes_from_services_map] list of n1ql nodes in cluster: [ip:10.112.205.101 port:8091 ssh_username:root] 2020-11-10 13:06:28 | INFO | MainProcess | test_thread | [newtuq.setUp] ip:10.112.205.101 port:8091 ssh_username:root 2020-11-10 13:06:28 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 30 secs. ... 2020-11-10 13:06:58 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:06:58 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:06:58 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:06:58 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:06:58 | INFO | MainProcess | test_thread | [collections_concurrent_indexes.setUp] ============== ConcurrentIndexes setup has started ============== 2020-11-10 13:06:59 | INFO | MainProcess | Cluster_Thread | [rest_client.create_bucket] http://10.112.205.101:8091/pools/default/buckets with param: name=test_bucket&ramQuotaMB=100&authType=none&replicaNumber=1&bucketType=membase&replicaIndex=1&threadsNumber=3&flushEnabled=1&evictionPolicy=valueOnly&compressionMode=passive&storageBackend=couchstore 2020-11-10 13:06:59 | INFO | MainProcess | Cluster_Thread | [rest_client.create_bucket] 0.01 seconds to create bucket test_bucket 2020-11-10 13:06:59 | INFO | MainProcess | Cluster_Thread | [bucket_helper.wait_for_memcached] waiting for memcached bucket : test_bucket in 10.112.205.101 to accept set ops 2020-11-10 13:07:00 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:07:01 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:07:01 | WARNING | MainProcess | Cluster_Thread | [task.check] vbucket map not ready after try 0 2020-11-10 13:07:01 | INFO | MainProcess | Cluster_Thread | [bucket_helper.wait_for_memcached] waiting for memcached bucket : test_bucket in 10.112.205.101 to accept set ops 2020-11-10 13:07:01 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:07:02 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:07:02 | WARNING | MainProcess | Cluster_Thread | [task.check] vbucket map not ready after try 1 2020-11-10 13:07:02 | INFO | MainProcess | Cluster_Thread | [bucket_helper.wait_for_memcached] waiting for memcached bucket : test_bucket in 10.112.205.101 to accept set ops 2020-11-10 13:07:02 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:07:02 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:07:02 | WARNING | MainProcess | Cluster_Thread | [task.check] vbucket map not ready after try 2 2020-11-10 13:07:02 | INFO | MainProcess | Cluster_Thread | [bucket_helper.wait_for_memcached] waiting for memcached bucket : test_bucket in 10.112.205.101 to accept set ops 2020-11-10 13:07:02 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:07:03 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:07:03 | WARNING | MainProcess | Cluster_Thread | [task.check] vbucket map not ready after try 3 2020-11-10 13:07:03 | INFO | MainProcess | Cluster_Thread | [bucket_helper.wait_for_memcached] waiting for memcached bucket : test_bucket in 10.112.205.101 to accept set ops 2020-11-10 13:07:03 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:07:03 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:07:03 | WARNING | MainProcess | Cluster_Thread | [task.check] vbucket map not ready after try 4 2020-11-10 13:07:03 | INFO | MainProcess | Cluster_Thread | [bucket_helper.wait_for_memcached] waiting for memcached bucket : test_bucket in 10.112.205.101 to accept set ops 2020-11-10 13:07:04 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:07:04 | INFO | MainProcess | Cluster_Thread | [data_helper.direct_client] creating direct client 10.112.205.101:11210 test_bucket 2020-11-10 13:07:04 | WARNING | MainProcess | Cluster_Thread | [task.check] vbucket map not ready after try 5 2020-11-10 13:07:04 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2020-11-10 13:07:04 | INFO | MainProcess | test_thread | [remote_util.ssh_connect_with_retries] SSH Connected to 10.112.205.101 as root 2020-11-10 13:07:04 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2020-11-10 13:07:05 | INFO | MainProcess | test_thread | [remote_util.extract_remote_info] extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2020-11-10 13:07:05 | INFO | MainProcess | test_thread | [collections_concurrent_indexes.setUp] ============== ConcurrentIndexes setup has completed ============== 2020-11-10 13:07:36 | INFO | MainProcess | test_thread | [rest_client.set_index_settings] {'indexer.scheduleCreateRetries': 2} set 2020-11-10 13:07:36 | INFO | MainProcess | test_thread | [basetestcase.get_nodes_from_services_map] list of index nodes in cluster: [ip:10.112.205.101 port:8091 ssh_username:root, ip:10.112.205.102 port:8091 ssh_username:root, ip:10.112.205.103 port:8091 ssh_username:root] 2020-11-10 13:07:36 | INFO | MainProcess | test_thread | [rest_client.create_scope] http://10.112.205.101:8091/pools/default/buckets/test_bucket/collections with params: name=test_scope_1 2020-11-10 13:07:36 | INFO | MainProcess | test_thread | [rest_client.create_scope] Scope created test_bucket->test_scope_1 {'uid': '1'} 2020-11-10 13:07:36 | INFO | MainProcess | test_thread | [rest_client._create_single_collection] http://10.112.205.101:8091/pools/default/buckets/test_bucket/collections/test_scope_1 with params: name=test_collection_1 2020-11-10 13:07:36 | INFO | MainProcess | test_thread | [rest_client.create_collection] Collection created test_bucket->test_scope_1->test_collection_1 manifest:{'uid': '2'} 2020-11-10 13:07:36 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 10 secs. Allowing time after collection creation ... 2020-11-10 13:07:46 | INFO | MainProcess | test_thread | [basetestcase.data_ops_javasdk_loader_in_batches] Number of batches : 10 2020-11-10 13:07:46 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 10 secs. ... 2020-11-10 13:07:56 | INFO | MainProcess | test_thread | [basetestcase.data_ops_javasdk_loader_in_batches] done 2020-11-10 13:07:56 | INFO | MainProcess | Cluster_Thread | [task.execute_for_collection] java -jar java_sdk_client/collections/target/javaclient/javaclient.jar -i 10.112.205.101 -u Administrator -p password -b test_bucket -s test_scope_1 -c test_collection_1 -n 10000 -pc 100 -pu 0 -pd 0 -l uniform -dsn 1 -dpx doc_ -dt Person -de 0 -ds 500 -ac False -st 0 -en 99999 2020-11-10 13:08:03 | INFO | MainProcess | Cluster_Thread | [task.execute_for_collection] java -jar java_sdk_client/collections/target/javaclient/javaclient.jar -i 10.112.205.101 -u Administrator -p password -b test_bucket -s test_scope_1 -c test_collection_1 -n 10000 -pc 100 -pu 0 -pd 0 -l uniform -dsn 10001 -dpx doc_ -dt Person -de 0 -ds 500 -ac False -st 0 -en 99999 2020-11-10 13:08:07 | INFO | MainProcess | Cluster_Thread | [task.execute_for_collection] java -jar java_sdk_client/collections/target/javaclient/javaclient.jar -i 10.112.205.101 -u Administrator -p password -b test_bucket -s test_scope_1 -c test_collection_1 -n 10000 -pc 100 -pu 0 -pd 0 -l uniform -dsn 20001 -dpx doc_ -dt Person -de 0 -ds 500 -ac False -st 0 -en 99999 2020-11-10 13:08:11 | INFO | MainProcess | Cluster_Thread | [task.execute_for_collection] java -jar java_sdk_client/collections/target/javaclient/javaclient.jar -i 10.112.205.101 -u Administrator -p password -b test_bucket -s test_scope_1 -c test_collection_1 -n 10000 -pc 100 -pu 0 -pd 0 -l uniform -dsn 30001 -dpx doc_ -dt Person -de 0 -ds 500 -ac False -st 0 -en 99999 2020-11-10 13:08:15 | INFO | MainProcess | Cluster_Thread | [task.execute_for_collection] java -jar java_sdk_client/collections/target/javaclient/javaclient.jar -i 10.112.205.101 -u Administrator -p password -b test_bucket -s test_scope_1 -c test_collection_1 -n 10000 -pc 100 -pu 0 -pd 0 -l uniform -dsn 40001 -dpx doc_ -dt Person -de 0 -ds 500 -ac False -st 0 -en 99999 2020-11-10 13:08:19 | INFO | MainProcess | Cluster_Thread | [task.execute_for_collection] java -jar java_sdk_client/collections/target/javaclient/javaclient.jar -i 10.112.205.101 -u Administrator -p password -b test_bucket -s test_scope_1 -c test_collection_1 -n 10000 -pc 100 -pu 0 -pd 0 -l uniform -dsn 50001 -dpx doc_ -dt Person -de 0 -ds 500 -ac False -st 0 -en 99999 2020-11-10 13:08:23 | INFO | MainProcess | Cluster_Thread | [task.execute_for_collection] java -jar java_sdk_client/collections/target/javaclient/javaclient.jar -i 10.112.205.101 -u Administrator -p password -b test_bucket -s test_scope_1 -c test_collection_1 -n 10000 -pc 100 -pu 0 -pd 0 -l uniform -dsn 60001 -dpx doc_ -dt Person -de 0 -ds 500 -ac False -st 0 -en 99999 2020-11-10 13:08:27 | INFO | MainProcess | Cluster_Thread | [task.execute_for_collection] java -jar java_sdk_client/collections/target/javaclient/javaclient.jar -i 10.112.205.101 -u Administrator -p password -b test_bucket -s test_scope_1 -c test_collection_1 -n 10000 -pc 100 -pu 0 -pd 0 -l uniform -dsn 70001 -dpx doc_ -dt Person -de 0 -ds 500 -ac False -st 0 -en 99999 2020-11-10 13:08:32 | INFO | MainProcess | Cluster_Thread | [task.execute_for_collection] java -jar java_sdk_client/collections/target/javaclient/javaclient.jar -i 10.112.205.101 -u Administrator -p password -b test_bucket -s test_scope_1 -c test_collection_1 -n 10000 -pc 100 -pu 0 -pd 0 -l uniform -dsn 80001 -dpx doc_ -dt Person -de 0 -ds 500 -ac False -st 0 -en 99999 2020-11-10 13:08:37 | INFO | MainProcess | Cluster_Thread | [task.execute_for_collection] java -jar java_sdk_client/collections/target/javaclient/javaclient.jar -i 10.112.205.101 -u Administrator -p password -b test_bucket -s test_scope_1 -c test_collection_1 -n 10000 -pc 100 -pu 0 -pd 0 -l uniform -dsn 90001 -dpx doc_ -dt Person -de 0 -ds 500 -ac False -st 0 -en 99999 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_0 | [tuq_helper.run_cbq_query] RUN QUERY CREATE INDEX idx_0 ON default:test_bucket.test_scope_1.test_collection_1(age) USING GSI WITH {'nodes': ['10.112.205.102:8091', '10.112.205.103:8091'], 'defer_build': True} 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_2 | [tuq_helper.run_cbq_query] RUN QUERY CREATE INDEX idx_2 ON default:test_bucket.test_scope_1.test_collection_1(country) USING GSI WITH {'nodes': ['10.112.205.102:8091', '10.112.205.103:8091'], 'defer_build': True} 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_3 | [tuq_helper.run_cbq_query] RUN QUERY CREATE INDEX idx_3 ON default:test_bucket.test_scope_1.test_collection_1(title) USING GSI WITH {'nodes': ['10.112.205.102:8091', '10.112.205.103:8091'], 'defer_build': True} 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_1 | [tuq_helper.run_cbq_query] RUN QUERY CREATE INDEX idx_1 ON default:test_bucket.test_scope_1.test_collection_1(city) USING GSI WITH {'nodes': ['10.112.205.102:8091', '10.112.205.103:8091'], 'defer_build': True} 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_3 | [rest_client.query_tool] query params : statement=CREATE+INDEX+idx_3+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28title%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_2 | [rest_client.query_tool] query params : statement=CREATE+INDEX+idx_2+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28country%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_1 | [rest_client.query_tool] query params : statement=CREATE+INDEX+idx_1+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28city%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_4 | [tuq_helper.run_cbq_query] RUN QUERY CREATE INDEX idx_4 ON default:test_bucket.test_scope_1.test_collection_1(firstName) USING GSI WITH {'nodes': ['10.112.205.102:8091', '10.112.205.103:8091'], 'defer_build': True} 2020-11-10 13:39:36 | INFO | MainProcess | test_thread | [basetestcase.sleep] sleep for 5 secs. ... 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_0 | [rest_client.query_tool] query params : statement=CREATE+INDEX+idx_0+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28age%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_7 | [tuq_helper.run_cbq_query] RUN QUERY CREATE INDEX idx_7 ON default:test_bucket.test_scope_1.test_collection_1(suffix) USING GSI WITH {'nodes': ['10.112.205.102:8091', '10.112.205.103:8091'], 'defer_build': True} 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_6 | [tuq_helper.run_cbq_query] RUN QUERY CREATE INDEX idx_6 ON default:test_bucket.test_scope_1.test_collection_1(streetAddress) USING GSI WITH {'nodes': ['10.112.205.102:8091', '10.112.205.103:8091'], 'defer_build': True} 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_5 | [tuq_helper.run_cbq_query] RUN QUERY CREATE INDEX idx_5 ON default:test_bucket.test_scope_1.test_collection_1(lastName) USING GSI WITH {'nodes': ['10.112.205.102:8091', '10.112.205.103:8091'], 'defer_build': True} 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_9 | [tuq_helper.run_cbq_query] RUN QUERY CREATE INDEX idx_9 ON default:test_bucket.test_scope_1.test_collection_1(phone) USING GSI WITH {'nodes': ['10.112.205.102:8091', '10.112.205.103:8091'], 'defer_build': True} 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_7 | [rest_client.query_tool] query params : statement=CREATE+INDEX+idx_7+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28suffix%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_8 | [tuq_helper.run_cbq_query] RUN QUERY CREATE INDEX idx_8 ON default:test_bucket.test_scope_1.test_collection_1(filler1) USING GSI WITH {'nodes': ['10.112.205.102:8091', '10.112.205.103:8091'], 'defer_build': True} 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_6 | [rest_client.query_tool] query params : statement=CREATE+INDEX+idx_6+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28streetAddress%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_4 | [rest_client.query_tool] query params : statement=CREATE+INDEX+idx_4+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28firstName%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_5 | [rest_client.query_tool] query params : statement=CREATE+INDEX+idx_5+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28lastName%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_8 | [rest_client.query_tool] query params : statement=CREATE+INDEX+idx_8+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28filler1%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_9 | [rest_client.query_tool] query params : statement=CREATE+INDEX+idx_9+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28phone%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D 2020-11-10 13:39:36 | ERROR | MainProcess | ThreadPoolExecutor-0_0 | [rest_client._http_request] POST http://10.112.205.101:8093/query?statement=CREATE+INDEX+idx_0+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28age%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 500 reason: unknown b'{\n"requestID": "427eb8ac-c43b-49df-b3f8-7ccf68eb0783",\n"signature": null,\n"results": [\n],\n"errors": [{"code":5000,"msg":"GSI CreateIndex() - cause: The index is scheduled for background creation. Index creation for index idx_0, bucket test_bucket, scope test_scope_1, collection test_collection_1 cannot start. Reason: Create index or Alter replica cannot proceed due to another concurrent create index request.."}],\n"status": "errors",\n"metrics": {"elapsedTime": "118.580401ms","executionTime": "118.559113ms","resultCount": 0,"resultSize": 0,"serviceLoad": 250,"errorCount": 1}\n}\n' auth: Administrator:password 2020-11-10 13:39:36 | ERROR | MainProcess | ThreadPoolExecutor-0_1 | [rest_client._http_request] POST http://10.112.205.101:8093/query?statement=CREATE+INDEX+idx_1+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28city%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 500 reason: unknown b'{\n"requestID": "cacac0d9-1355-4255-861d-0d4a227aacea",\n"signature": null,\n"results": [\n],\n"errors": [{"code":5000,"msg":"GSI CreateIndex() - cause: The index is scheduled for background creation. Index creation for index idx_1, bucket test_bucket, scope test_scope_1, collection test_collection_1 cannot start. Reason: Create index or Alter replica cannot proceed due to another concurrent create index request.."}],\n"status": "errors",\n"metrics": {"elapsedTime": "129.665219ms","executionTime": "129.558066ms","resultCount": 0,"resultSize": 0,"serviceLoad": 200,"errorCount": 1}\n}\n' auth: Administrator:password 2020-11-10 13:39:36 | ERROR | MainProcess | ThreadPoolExecutor-0_2 | [rest_client._http_request] POST http://10.112.205.101:8093/query?statement=CREATE+INDEX+idx_2+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28country%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 500 reason: unknown b'{\n"requestID": "4d98089c-570d-402d-a2d0-32d5142162bb",\n"signature": null,\n"results": [\n],\n"errors": [{"code":5000,"msg":"GSI CreateIndex() - cause: The index is scheduled for background creation. Index creation for index idx_2, bucket test_bucket, scope test_scope_1, collection test_collection_1 cannot start. Reason: Create index or Alter replica cannot proceed due to another concurrent create index request.."}],\n"status": "errors",\n"metrics": {"elapsedTime": "119.559549ms","executionTime": "119.527483ms","resultCount": 0,"resultSize": 0,"serviceLoad": 225,"errorCount": 1}\n}\n' auth: Administrator:password 2020-11-10 13:39:36 | INFO | MainProcess | ThreadPoolExecutor-0_3 | [tuq_helper.run_cbq_query] TOTAL ELAPSED TIME: 252.321344ms 2020-11-10 13:39:36 | ERROR | MainProcess | ThreadPoolExecutor-0_6 | [rest_client._http_request] POST http://10.112.205.101:8093/query?statement=CREATE+INDEX+idx_6+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28streetAddress%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 500 reason: unknown b'{\n"requestID": "38573022-98e9-48bb-923c-963ecaf41084",\n"signature": null,\n"results": [\n],\n"errors": [{"code":5000,"msg":"GSI CreateIndex() - cause: The index is scheduled for background creation. Index creation for index idx_6, bucket test_bucket, scope test_scope_1, collection test_collection_1 cannot start. Reason: Create index or Alter replica cannot proceed due to another concurrent create index request.."}],\n"status": "errors",\n"metrics": {"elapsedTime": "303.819443ms","executionTime": "221.389722ms","resultCount": 0,"resultSize": 0,"serviceLoad": 150,"errorCount": 1}\n}\n' auth: Administrator:password 2020-11-10 13:39:36 | ERROR | MainProcess | ThreadPoolExecutor-0_8 | [rest_client._http_request] POST http://10.112.205.101:8093/query?statement=CREATE+INDEX+idx_8+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28filler1%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 500 reason: unknown b'{\n"requestID": "8ac60aba-352d-4213-99e6-bf906abdc504",\n"signature": null,\n"results": [\n],\n"errors": [{"code":5000,"msg":"GSI CreateIndex() - cause: The index is scheduled for background creation. Index creation for index idx_8, bucket test_bucket, scope test_scope_1, collection test_collection_1 cannot start. Reason: Create index or Alter replica cannot proceed due to another concurrent create index request.."}],\n"status": "errors",\n"metrics": {"elapsedTime": "325.672533ms","executionTime": "244.18765ms","resultCount": 0,"resultSize": 0,"serviceLoad": 125,"errorCount": 1}\n}\n' auth: Administrator:password 2020-11-10 13:39:36 | ERROR | MainProcess | ThreadPoolExecutor-0_7 | [rest_client._http_request] POST http://10.112.205.101:8093/query?statement=CREATE+INDEX+idx_7+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28suffix%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 500 reason: unknown b'{\n"requestID": "a504b1e4-3523-4347-a2f0-745412884b2c",\n"signature": null,\n"results": [\n],\n"errors": [{"code":5000,"msg":"GSI CreateIndex() - cause: The index is scheduled for background creation. Index creation for index idx_7, bucket test_bucket, scope test_scope_1, collection test_collection_1 cannot start. Reason: Create index or Alter replica cannot proceed due to another concurrent create index request.."}],\n"status": "errors",\n"metrics": {"elapsedTime": "325.845823ms","executionTime": "241.360211ms","resultCount": 0,"resultSize": 0,"serviceLoad": 100,"errorCount": 1}\n}\n' auth: Administrator:password 2020-11-10 13:39:36 | ERROR | MainProcess | ThreadPoolExecutor-0_9 | [rest_client._http_request] POST http://10.112.205.101:8093/query?statement=CREATE+INDEX+idx_9+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28phone%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 500 reason: unknown b'{\n"requestID": "80ed09e5-9aa4-4584-ace1-01e25ff7045a",\n"signature": null,\n"results": [\n],\n"errors": [{"code":5000,"msg":"GSI CreateIndex() - cause: The index is scheduled for background creation. Index creation for index idx_9, bucket test_bucket, scope test_scope_1, collection test_collection_1 cannot start. Reason: Create index or Alter replica cannot proceed due to another concurrent create index request.."}],\n"status": "errors",\n"metrics": {"elapsedTime": "599.809258ms","executionTime": "279.38653ms","resultCount": 0,"resultSize": 0,"serviceLoad": 75,"errorCount": 1}\n}\n' auth: Administrator:password 2020-11-10 13:39:37 | INFO | MainProcess | ThreadPoolExecutor-0_4 | [tuq_helper.run_cbq_query] TOTAL ELAPSED TIME: 984.43176ms 2020-11-10 13:39:39 | ERROR | MainProcess | ThreadPoolExecutor-0_5 | [rest_client._http_request] POST http://10.112.205.101:8093/query?statement=CREATE+INDEX+idx_5+ON+default%3Atest_bucket.test_scope_1.test_collection_1%28lastName%29+USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.102%3A8091%27%2C+%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 500 reason: unknown b'{\n"requestID": "d0398a99-17a9-42cd-8b16-bc907167968b",\n"signature": null,\n"results": [\n],\n"errors": [{"code":5000,"msg":"GSI CreateIndex() - cause: The index is scheduled for background creation. Index creation for index idx_5, bucket test_bucket, scope test_scope_1, collection test_collection_1 cannot start. Reason: Create index or Alter replica cannot proceed due to another concurrent create index request.."}],\n"status": "errors",\n"metrics": {"elapsedTime": "3.542324992s","executionTime": "3.238267905s","resultCount": 0,"resultSize": 0,"serviceLoad": 25,"errorCount": 1}\n}\n' auth: Administrator:password 2020-11-10 13:39:41 | INFO | MainProcess | Cluster_Thread | [rest_client.rebalance] rebalance params : {'knownNodes': 'ns_1@10.112.205.101,ns_1@10.112.205.102,ns_1@10.112.205.103', 'ejectedNodes': 'ns_1@10.112.205.103', 'user': 'Administrator', 'password': 'password'} 2020-11-10 13:39:41 | INFO | MainProcess | Cluster_Thread | [rest_client.rebalance] rebalance operation started 2020-11-10 13:39:41 | INFO | MainProcess | Cluster_Thread | [rest_client._rebalance_status_and_progress] rebalance percentage : 0.00 % 2020-11-10 13:39:41 | INFO | MainProcess | Cluster_Thread | [task.check] Rebalance - status: running, progress: 0.00% 2020-11-10 13:39:51 | INFO | MainProcess | Cluster_Thread | [task.check] Rebalance - status: none, progress: 100.00% 2020-11-10 13:39:51 | INFO | MainProcess | Cluster_Thread | [task.check] rebalancing was completed with progress: 100% in 10.068469047546387 sec [{'defnId': 6689539003682678602, 'instId': 6689539003682678602, 'name': 'idx_0', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`age`'], 'indexType': 'plasma', 'status': 'Error', 'definition': 'CREATE INDEX `idx_0` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`age`) WITH { "defer_build":true, "nodes":[ "10.112.205.102:8091","10.112.205.103:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.102:8091'], 'error': 'Indexer node (10.112.205.103:8091) not found. The node may be failed or under rebalance or network partitioned from query process.', 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': None, 'numReplica': 1, 'indexName': 'idx_0', 'replicaId': 0, 'stale': False, 'lastScanTime': 'NA'}, {'defnId': 17748990959476978168, 'instId': 10075227978448267544, 'name': 'idx_1', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`city`'], 'indexType': 'plasma', 'status': 'Created', 'definition': 'CREATE INDEX `idx_1` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`city`) WITH { "defer_build":true, "nodes":[ "10.112.205.101:8091","10.112.205.102:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.102:8091'], 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': {'10.112.205.102:8091': [0]}, 'numReplica': 1, 'indexName': 'idx_1', 'replicaId': 0, 'stale': False, 'lastScanTime': 'NA'}, {'defnId': 17748990959476978168, 'instId': 2386135223686407325, 'name': 'idx_1 (replica 1)', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`city`'], 'indexType': 'plasma', 'status': 'Created', 'definition': 'CREATE INDEX `idx_1` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`city`) WITH { "defer_build":true, "nodes":[ "10.112.205.101:8091","10.112.205.102:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.101:8091'], 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': {'10.112.205.101:8091': [0]}, 'numReplica': 1, 'indexName': 'idx_1', 'replicaId': 1, 'stale': False, 'lastScanTime': 'NA'}, {'defnId': 2608469180817333143, 'instId': 10407662669178576020, 'name': 'idx_2', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`country`'], 'indexType': 'plasma', 'status': 'Created', 'definition': 'CREATE INDEX `idx_2` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`country`) WITH { "defer_build":true, "nodes":[ "10.112.205.101:8091","10.112.205.102:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.101:8091'], 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': {'10.112.205.101:8091': [0]}, 'numReplica': 1, 'indexName': 'idx_2', 'replicaId': 0, 'stale': False, 'lastScanTime': 'NA'}, {'defnId': 2608469180817333143, 'instId': 16932543357536764423, 'name': 'idx_2 (replica 1)', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`country`'], 'indexType': 'plasma', 'status': 'Created', 'definition': 'CREATE INDEX `idx_2` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`country`) WITH { "defer_build":true, "nodes":[ "10.112.205.101:8091","10.112.205.102:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.102:8091'], 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': {'10.112.205.102:8091': [0]}, 'numReplica': 1, 'indexName': 'idx_2', 'replicaId': 1, 'stale': False, 'lastScanTime': 'NA'}, {'defnId': 14144530508425990434, 'instId': 5573008005268669552, 'name': 'idx_3', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`title`'], 'indexType': 'plasma', 'status': 'Created', 'definition': 'CREATE INDEX `idx_3` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`title`) WITH { "defer_build":true, "nodes":[ "10.112.205.101:8091","10.112.205.102:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.102:8091'], 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': {'10.112.205.102:8091': [0]}, 'numReplica': 1, 'indexName': 'idx_3', 'replicaId': 0, 'stale': False, 'lastScanTime': 'NA'}, {'defnId': 14144530508425990434, 'instId': 11123823218909581487, 'name': 'idx_3 (replica 1)', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`title`'], 'indexType': 'plasma', 'status': 'Created', 'definition': 'CREATE INDEX `idx_3` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`title`) WITH { "defer_build":true, "nodes":[ "10.112.205.101:8091","10.112.205.102:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.101:8091'], 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': {'10.112.205.101:8091': [0]}, 'numReplica': 1, 'indexName': 'idx_3', 'replicaId': 1, 'stale': False, 'lastScanTime': 'NA'}, {'defnId': 14353033841875818748, 'instId': 13509766804183786616, 'name': 'idx_4', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`firstName`'], 'indexType': 'plasma', 'status': 'Created', 'definition': 'CREATE INDEX `idx_4` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`firstName`) WITH { "defer_build":true, "nodes":[ "10.112.205.101:8091","10.112.205.102:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.101:8091'], 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': {'10.112.205.101:8091': [0]}, 'numReplica': 1, 'indexName': 'idx_4', 'replicaId': 0, 'stale': False, 'lastScanTime': 'NA'}, {'defnId': 14353033841875818748, 'instId': 11108741149353718725, 'name': 'idx_4 (replica 1)', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`firstName`'], 'indexType': 'plasma', 'status': 'Created', 'definition': 'CREATE INDEX `idx_4` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`firstName`) WITH { "defer_build":true, "nodes":[ "10.112.205.101:8091","10.112.205.102:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.102:8091'], 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': {'10.112.205.102:8091': [0]}, 'numReplica': 1, 'indexName': 'idx_4', 'replicaId': 1, 'stale': False, 'lastScanTime': 'NA'}, {'defnId': 9399201848251869725, 'instId': 9399201848251869725, 'name': 'idx_5', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`lastName`'], 'indexType': 'plasma', 'status': 'Error', 'definition': 'CREATE INDEX `idx_5` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`lastName`) WITH { "defer_build":true, "nodes":[ "10.112.205.102:8091","10.112.205.103:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.102:8091'], 'error': 'Indexer node (10.112.205.103:8091) not found. The node may be failed or under rebalance or network partitioned from query process.', 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': None, 'numReplica': 1, 'indexName': 'idx_5', 'replicaId': 0, 'stale': False, 'lastScanTime': 'NA'}, {'defnId': 580715957121772959, 'instId': 580715957121772959, 'name': 'idx_6', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`streetAddress`'], 'indexType': 'plasma', 'status': 'Error', 'definition': 'CREATE INDEX `idx_6` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`streetAddress`) WITH { "defer_build":true, "nodes":[ "10.112.205.102:8091","10.112.205.103:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.102:8091'], 'error': 'Indexer node (10.112.205.103:8091) not found. The node may be failed or under rebalance or network partitioned from query process.', 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': None, 'numReplica': 1, 'indexName': 'idx_6', 'replicaId': 0, 'stale': False, 'lastScanTime': 'NA'}, {'defnId': 12620757198349048561, 'instId': 12620757198349048561, 'name': 'idx_7', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`suffix`'], 'indexType': 'plasma', 'status': 'Error', 'definition': 'CREATE INDEX `idx_7` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`suffix`) WITH { "defer_build":true, "nodes":[ "10.112.205.102:8091","10.112.205.103:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.102:8091'], 'error': 'Indexer node (10.112.205.103:8091) not found. The node may be failed or under rebalance or network partitioned from query process.', 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': None, 'numReplica': 1, 'indexName': 'idx_7', 'replicaId': 0, 'stale': False, 'lastScanTime': 'NA'}, {'defnId': 5398079839267784363, 'instId': 5398079839267784363, 'name': 'idx_8', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`filler1`'], 'indexType': 'plasma', 'status': 'Error', 'definition': 'CREATE INDEX `idx_8` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`filler1`) WITH { "defer_build":true, "nodes":[ "10.112.205.102:8091","10.112.205.103:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.102:8091'], 'error': 'Indexer node (10.112.205.103:8091) not found. The node may be failed or under rebalance or network partitioned from query process.', 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': None, 'numReplica': 1, 'indexName': 'idx_8', 'replicaId': 0, 'stale': False, 'lastScanTime': 'NA'}, {'defnId': 705950630639120636, 'instId': 705950630639120636, 'name': 'idx_9', 'bucket': 'test_bucket', 'scope': 'test_scope_1', 'collection': 'test_collection_1', 'secExprs': ['`phone`'], 'indexType': 'plasma', 'status': 'Error', 'definition': 'CREATE INDEX `idx_9` ON `test_bucket`.`test_scope_1`.`test_collection_1`(`phone`) WITH { "defer_build":true, "nodes":[ "10.112.205.102:8091","10.112.205.103:8091" ], "num_replica":1 }', 'hosts': ['10.112.205.102:8091'], 'error': 'Indexer node (10.112.205.103:8091) not found. The node may be failed or under rebalance or network partitioned from query process.', 'completion': 0, 'progress': 0, 'scheduled': False, 'partitioned': False, 'numPartition': 1, 'partitionMap': None, 'numReplica': 1, 'indexName': 'idx_9', 'replicaId': 0, 'stale': False, 'lastScanTime': 'NA'}]