/Users/lfc/venv37/bin/python "/Applications/PyCharm CE.app/Contents/plugins/python-ce/helpers/pydev/pydevd.py" --multiproc --qt-support=auto --client 127.0.0.1 --port 50359 --file /Users/lfc/testrunner/testrunner.py -i b/resources/tmp.ini -p gsi_type=plasma,initial_version=6.6.5-10080,upgrade_to=7.1.0-2445,get-cbcollect-info=True,java_sdk_client=True,reset_services=True,get-cbcollect-info=True,bucket_storage=couchstore -t gsi.upgrade_gsi.UpgradeSecondaryIndex.test_online_upgrade,before=create_index,in_between=query:verify_query_result-query_with_explain,after=query:verify_query_result-drop_index,doc_ops=True,update_ops_per=.5,nodes_out=1,nodes_out_dist=index:1,standard_buckets=1,dataset=default,groups=simple,nodes_init=4,services_init=kv:n1ql-kv-index-index,doc-per-day=10,init_nodes=False,gsi_type=plasma,initial_version=6.6.5-10080,upgrade_to=7.1.0-2445,get-cbcollect-info=True,java_sdk_client=True,reset_services=True,bucket_storage=couchstore Connected to pydev debugger (build 212.4746.96) INFO:root:__main__ *** TestRunner *** INFO:__main__:TestRunner: parsing args... INFO:__main__:Checking arguments... INFO:__main__:TestRunner: start... INFO:__main__:Global Test input params: {'bucket_storage': 'couchstore', 'cluster_name': 'tmp', 'get-cbcollect-info': 'True', 'gsi_type': 'plasma', 'ini': 'b/resources/tmp.ini', 'initial_version': '6.6.5-10080', 'java_sdk_client': 'True', 'num_nodes': 4, 'reset_services': 'True', 'upgrade_to': '7.1.0-2445'} INFO:__main__: Number of tests initially selected before GROUP filters: 1 INFO:__main__:--> Running test: gsi.upgrade_gsi.UpgradeSecondaryIndex.test_online_upgrade,before=create_index,in_between=query:verify_query_result-query_with_explain,after=query:verify_query_result-drop_index,doc_ops=True,update_ops_per=.5,nodes_out=1,nodes_out_dist=index:1,standard_buckets=1,dataset=default,groups=simple,nodes_init=4,services_init=kv:n1ql-kv-index-index,doc-per-day=10,init_nodes=False,gsi_type=plasma,initial_version=6.6.5-10080,upgrade_to=7.1.0-2445,get-cbcollect-info=True,java_sdk_client=True,reset_services=True,bucket_storage=couchstore INFO:__main__:Logs folder: /Users/lfc/testrunner/logs/testrunner-22-Mar-09_13-52-27/test_1 Logs will be stored at /Users/lfc/testrunner/logs/testrunner-22-Mar-09_13-52-27/test_1 ./testrunner -i b/resources/tmp.ini -p gsi_type=plasma,initial_version=6.6.5-10080,upgrade_to=7.1.0-2445,get-cbcollect-info=True,java_sdk_client=True,reset_services=True,get-cbcollect-info=True,bucket_storage=couchstore -t gsi.upgrade_gsi.UpgradeSecondaryIndex.test_online_upgrade,before=create_index,in_between=query:verify_query_result-query_with_explain,after=query:verify_query_result-drop_index,doc_ops=True,update_ops_per=.5,nodes_out=1,nodes_out_dist=index:1,standard_buckets=1,dataset=default,groups=simple,nodes_init=4,services_init=kv:n1ql-kv-index-index,doc-per-day=10,init_nodes=False,gsi_type=plasma,initial_version=6.6.5-10080,upgrade_to=7.1.0-2445,get-cbcollect-info=True,java_sdk_client=True,reset_services=True,bucket_storage=couchstore Test Input params: {'before': 'create_index', 'in_between': 'query:verify_query_result-query_with_explain', 'after': 'query:verify_query_result-drop_index', 'doc_ops': 'True', 'update_ops_per': '.5', 'nodes_out': '1', 'nodes_out_dist': 'index:1', 'standard_buckets': '1', 'dataset': 'default', 'groups': 'simple', 'nodes_init': '4', 'services_init': 'kv:n1ql-kv-index-index', 'doc-per-day': '10', 'init_nodes': 'False', 'gsi_type': 'plasma', 'initial_version': '6.6.5-10080', 'upgrade_to': '7.1.0-2445', 'get-cbcollect-info': 'True', 'java_sdk_client': 'True', 'reset_services': 'True', 'bucket_storage': 'couchstore', 'ini': 'b/resources/tmp.ini', 'cluster_name': 'tmp', 'num_nodes': 4, 'case_number': 1, 'last_case_fail': 'False', 'teardown_run': 'False', 'logs_folder': '/Users/lfc/testrunner/logs/testrunner-22-Mar-09_13-52-27/test_1'} test_online_upgrade (gsi.upgrade_gsi.UpgradeSecondaryIndex) ... 2022-03-09 13:52:28,844 - root - INFO - ============== EventingBaseTest setup has started ============== 2022-03-09 13:52:28,847 - root - INFO - Starting Test: test_online_upgrade 2022-03-09 13:52:28,852 - root - INFO - SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2022-03-09 13:52:28,971 - root - INFO - SSH Connected to 10.112.205.101 as root 2022-03-09 13:52:29,071 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:52:29,333 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:52:29,350 - root - INFO - -->is_ns_server_running? 2022-03-09 13:52:29,373 - root - INFO - Node version in cluster 6.6.5-10080-enterprise 2022-03-09 13:52:29,375 - root - INFO - ============== basetestcase setup was started for test #1 test_online_upgrade============== 2022-03-09 13:52:29,387 - root - ERROR - GET http://10.112.205.101:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2022-03-09 13:52:29,413 - root - ERROR - GET http://10.112.205.101:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2022-03-09 13:52:29,415 - root - INFO - cannot find service node index in cluster 2022-03-09 13:52:29,420 - root - INFO - Building docker image with java sdk client 2022-03-09 13:52:35,693 - root - INFO - initializing cluster 2022-03-09 13:52:35,699 - root - INFO - SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2022-03-09 13:52:35,758 - root - INFO - SSH Connected to 10.112.205.101 as root 2022-03-09 13:52:35,849 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:52:36,105 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:52:36,116 - root - INFO - SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2022-03-09 13:52:36,212 - root - INFO - SSH Connected to 10.112.205.101 as root 2022-03-09 13:52:36,302 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:52:36,559 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:52:36,610 - root - INFO - 10.112.205.101 **** The linux version file /opt/couchbase/ VERSION.txt exists 2022-03-09 13:52:36,682 - root - INFO - Running systemd command on this server 2022-03-09 13:52:36,683 - root - INFO - running command.raw on 10.112.205.101: systemctl stop couchbase-server.service 2022-03-09 13:52:38,264 - root - INFO - command executed successfully with root 2022-03-09 13:52:38,266 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:52:38,322 - root - INFO - Couchbase stopped 2022-03-09 13:52:38,324 - root - INFO - running command.raw on 10.112.205.101: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2022-03-09 13:52:38,336 - root - INFO - command executed successfully with root 2022-03-09 13:52:38,337 - root - INFO - running command.raw on 10.112.205.101: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2022-03-09 13:52:38,383 - root - INFO - command executed successfully with root 2022-03-09 13:52:38,387 - root - INFO - SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2022-03-09 13:52:38,447 - root - INFO - SSH Connected to 10.112.205.101 as root 2022-03-09 13:52:38,537 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:52:38,797 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:52:38,850 - root - INFO - 10.112.205.101 **** The linux version file /opt/couchbase/ VERSION.txt exists 2022-03-09 13:52:38,851 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:52:38,902 - root - INFO - Starting couchbase server 2022-03-09 13:52:39,014 - root - INFO - Running systemd command on this server 2022-03-09 13:52:39,015 - root - INFO - running command.raw on 10.112.205.101: systemctl start couchbase-server.service 2022-03-09 13:52:39,028 - root - INFO - command executed successfully with root 2022-03-09 13:52:39,030 - root - INFO - 10.112.205.101:sleep for 5 secs. waiting for couchbase server to come up ... 2022-03-09 13:52:44,033 - root - INFO - running command.raw on 10.112.205.101: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2022-03-09 13:52:44,045 - root - INFO - command executed successfully with root 2022-03-09 13:52:44,047 - root - INFO - Couchbase server status: [' Process: 7512 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2022-03-09 13:52:44,048 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:52:44,099 - root - INFO - process beam.smp is running on 10.112.205.101: with pid 7577 2022-03-09 13:52:44,100 - root - INFO - Couchbase started 2022-03-09 13:52:44,103 - root - INFO - SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2022-03-09 13:52:44,229 - root - INFO - SSH Connected to 10.112.205.102 as root 2022-03-09 13:52:44,340 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:52:44,670 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:52:44,683 - root - INFO - SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2022-03-09 13:52:44,739 - root - INFO - SSH Connected to 10.112.205.102 as root 2022-03-09 13:52:44,826 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:52:45,119 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:52:45,171 - root - INFO - 10.112.205.102 **** The linux version file /opt/couchbase/ VERSION.txt exists 2022-03-09 13:52:45,239 - root - INFO - Running systemd command on this server 2022-03-09 13:52:45,240 - root - INFO - running command.raw on 10.112.205.102: systemctl stop couchbase-server.service 2022-03-09 13:52:46,804 - root - INFO - command executed successfully with root 2022-03-09 13:52:46,805 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:52:46,818 - root - INFO - Couchbase stopped 2022-03-09 13:52:46,820 - root - INFO - running command.raw on 10.112.205.102: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2022-03-09 13:52:46,830 - root - INFO - command executed successfully with root 2022-03-09 13:52:46,831 - root - INFO - running command.raw on 10.112.205.102: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2022-03-09 13:52:46,876 - root - INFO - command executed successfully with root 2022-03-09 13:52:46,879 - root - INFO - SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2022-03-09 13:52:47,007 - root - INFO - SSH Connected to 10.112.205.102 as root 2022-03-09 13:52:47,089 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:52:47,344 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:52:47,395 - root - INFO - 10.112.205.102 **** The linux version file /opt/couchbase/ VERSION.txt exists 2022-03-09 13:52:47,395 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:52:47,447 - root - INFO - Starting couchbase server 2022-03-09 13:52:47,555 - root - INFO - Running systemd command on this server 2022-03-09 13:52:47,555 - root - INFO - running command.raw on 10.112.205.102: systemctl start couchbase-server.service 2022-03-09 13:52:47,576 - root - INFO - command executed successfully with root 2022-03-09 13:52:47,578 - root - INFO - 10.112.205.102:sleep for 5 secs. waiting for couchbase server to come up ... 2022-03-09 13:52:52,582 - root - INFO - running command.raw on 10.112.205.102: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2022-03-09 13:52:52,596 - root - INFO - command executed successfully with root 2022-03-09 13:52:52,598 - root - INFO - Couchbase server status: [' Process: 7307 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2022-03-09 13:52:52,599 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:52:52,649 - root - INFO - process beam.smp is running on 10.112.205.102: with pid 7373 2022-03-09 13:52:52,650 - root - INFO - Couchbase started 2022-03-09 13:52:52,653 - root - INFO - SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2022-03-09 13:52:52,725 - root - INFO - SSH Connected to 10.112.205.103 as root 2022-03-09 13:52:52,830 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:52:53,120 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:52:53,133 - root - INFO - SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2022-03-09 13:52:53,190 - root - INFO - SSH Connected to 10.112.205.103 as root 2022-03-09 13:52:53,273 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:52:53,523 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:52:53,575 - root - INFO - 10.112.205.103 **** The linux version file /opt/couchbase/ VERSION.txt exists 2022-03-09 13:52:53,683 - root - INFO - Running systemd command on this server 2022-03-09 13:52:53,684 - root - INFO - running command.raw on 10.112.205.103: systemctl stop couchbase-server.service 2022-03-09 13:52:55,201 - root - INFO - command executed successfully with root 2022-03-09 13:52:55,202 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:52:55,214 - root - INFO - Couchbase stopped 2022-03-09 13:52:55,215 - root - INFO - running command.raw on 10.112.205.103: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2022-03-09 13:52:55,225 - root - INFO - command executed successfully with root 2022-03-09 13:52:55,226 - root - INFO - running command.raw on 10.112.205.103: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2022-03-09 13:52:55,274 - root - INFO - command executed successfully with root 2022-03-09 13:52:55,277 - root - INFO - SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2022-03-09 13:52:55,335 - root - INFO - SSH Connected to 10.112.205.103 as root 2022-03-09 13:52:55,416 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:52:55,666 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:52:55,717 - root - INFO - 10.112.205.103 **** The linux version file /opt/couchbase/ VERSION.txt exists 2022-03-09 13:52:55,718 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:52:55,772 - root - INFO - Starting couchbase server 2022-03-09 13:52:55,919 - root - INFO - Running systemd command on this server 2022-03-09 13:52:55,920 - root - INFO - running command.raw on 10.112.205.103: systemctl start couchbase-server.service 2022-03-09 13:52:55,934 - root - INFO - command executed successfully with root 2022-03-09 13:52:55,935 - root - INFO - 10.112.205.103:sleep for 5 secs. waiting for couchbase server to come up ... 2022-03-09 13:53:00,942 - root - INFO - running command.raw on 10.112.205.103: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2022-03-09 13:53:00,955 - root - INFO - command executed successfully with root 2022-03-09 13:53:00,956 - root - INFO - Couchbase server status: [' Process: 7181 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2022-03-09 13:53:00,957 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:53:01,008 - root - INFO - process beam.smp is running on 10.112.205.103: with pid 7247 2022-03-09 13:53:01,009 - root - INFO - Couchbase started 2022-03-09 13:53:01,013 - root - INFO - SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2022-03-09 13:53:01,088 - root - INFO - SSH Connected to 10.112.205.104 as root 2022-03-09 13:53:01,190 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:53:01,482 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:53:01,493 - root - INFO - SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2022-03-09 13:53:01,549 - root - INFO - SSH Connected to 10.112.205.104 as root 2022-03-09 13:53:01,632 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:53:01,883 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:53:01,934 - root - INFO - 10.112.205.104 **** The linux version file /opt/couchbase/ VERSION.txt exists 2022-03-09 13:53:02,043 - root - INFO - Running systemd command on this server 2022-03-09 13:53:02,044 - root - INFO - running command.raw on 10.112.205.104: systemctl stop couchbase-server.service 2022-03-09 13:53:03,570 - root - INFO - command executed successfully with root 2022-03-09 13:53:03,571 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:53:03,624 - root - INFO - Couchbase stopped 2022-03-09 13:53:03,626 - root - INFO - running command.raw on 10.112.205.104: rm -rf /opt/couchbase/var/lib/couchbase/data/* 2022-03-09 13:53:03,636 - root - INFO - command executed successfully with root 2022-03-09 13:53:03,637 - root - INFO - running command.raw on 10.112.205.104: rm -rf /opt/couchbase/var/lib/couchbase/config/* 2022-03-09 13:53:03,685 - root - INFO - command executed successfully with root 2022-03-09 13:53:03,705 - root - INFO - SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2022-03-09 13:53:03,813 - root - INFO - SSH Connected to 10.112.205.104 as root 2022-03-09 13:53:03,896 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:53:04,146 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:53:04,197 - root - INFO - 10.112.205.104 **** The linux version file /opt/couchbase/ VERSION.txt exists 2022-03-09 13:53:04,211 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:53:04,248 - root - INFO - Starting couchbase server 2022-03-09 13:53:04,314 - root - INFO - Running systemd command on this server 2022-03-09 13:53:04,315 - root - INFO - running command.raw on 10.112.205.104: systemctl start couchbase-server.service 2022-03-09 13:53:04,328 - root - INFO - command executed successfully with root 2022-03-09 13:53:04,329 - root - INFO - 10.112.205.104:sleep for 5 secs. waiting for couchbase server to come up ... 2022-03-09 13:53:09,332 - root - INFO - running command.raw on 10.112.205.104: systemctl status couchbase-server.service | grep ExecStop=/opt/couchbase/bin/couchbase-server 2022-03-09 13:53:09,345 - root - INFO - command executed successfully with root 2022-03-09 13:53:09,346 - root - INFO - Couchbase server status: [' Process: 7300 ExecStop=/opt/couchbase/bin/couchbase-server -k (code=exited, status=0/SUCCESS)'] 2022-03-09 13:53:09,346 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:53:09,398 - root - INFO - process beam.smp is running on 10.112.205.104: with pid 7366 2022-03-09 13:53:09,399 - root - INFO - Couchbase started 2022-03-09 13:53:11,555 - root - INFO - server: ip:10.112.205.101 port:8091 ssh_username:root, nodes/self 2022-03-09 13:53:11,560 - root - INFO - {'uptime': '29', 'memoryTotal': 1930829824, 'memoryFree': 1583800320, 'mcdMemoryReserved': 1473, 'mcdMemoryAllocated': 1473, 'status': 'healthy', 'hostname': '10.112.205.101:8091', 'clusterCompatibility': 393222, 'clusterMembership': 'active', 'recoveryType': 'none', 'version': '6.6.5-10080-enterprise', 'os': 'x86_64-unknown-linux-gnu', 'ports': [], 'availableStorage': [], 'storage': [], 'memoryQuota': 256, 'memcached': 11210, 'id': 'ns_1@10.112.205.101', 'ip': '10.112.205.101', 'internal_ip': '', 'rest_username': '', 'rest_password': '', 'port': '8091', 'services': ['kv'], 'storageTotalRam': 1841, 'curr_items': 0} 2022-03-09 13:53:11,563 - root - ERROR - GET http://10.112.205.101:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2022-03-09 13:53:11,572 - root - INFO - pools/default params : memoryQuota=986 2022-03-09 13:53:11,578 - root - INFO - --> init_node_services(Administrator,password,10.112.205.101,8091,['kv', 'n1ql']) 2022-03-09 13:53:11,579 - root - INFO - /node/controller/setupServices params on 10.112.205.101: 8091:hostname=10.112.205.101&user=Administrator&password=password&services=kv%2Cn1ql 2022-03-09 13:53:11,589 - root - INFO - --> in init_cluster...Administrator,password,8091 2022-03-09 13:53:11,590 - root - INFO - settings/web params on 10.112.205.101:8091:port=8091&username=Administrator&password=password 2022-03-09 13:53:11,633 - root - INFO - --> status:True 2022-03-09 13:53:11,635 - root - INFO - SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2022-03-09 13:53:11,765 - root - INFO - SSH Connected to 10.112.205.101 as root 2022-03-09 13:53:12,019 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:53:12,473 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:53:12,475 - root - INFO - running command.raw on 10.112.205.101: curl --silent --show-error http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2022-03-09 13:53:12,684 - root - INFO - command executed successfully with root 2022-03-09 13:53:12,685 - root - INFO - ['ok'] 2022-03-09 13:53:12,693 - root - INFO - /diag/eval status on 10.112.205.101:8091: True content: [6,6] command: cluster_compat_mode:get_compat_version(). 2022-03-09 13:53:12,697 - root - INFO - /diag/eval status on 10.112.205.101:8091: True content: [6,6] command: cluster_compat_mode:get_compat_version(). 2022-03-09 13:53:12,702 - root - INFO - settings/indexes params : storageMode=plasma 2022-03-09 13:53:12,722 - root - INFO - server: ip:10.112.205.102 port:8091 ssh_username:root, nodes/self 2022-03-09 13:53:12,726 - root - INFO - {'uptime': '22', 'memoryTotal': 2033594368, 'memoryFree': 1678827520, 'mcdMemoryReserved': 1551, 'mcdMemoryAllocated': 1551, 'status': 'healthy', 'hostname': '10.112.205.102:8091', 'clusterCompatibility': 393222, 'clusterMembership': 'active', 'recoveryType': 'none', 'version': '6.6.5-10080-enterprise', 'os': 'x86_64-unknown-linux-gnu', 'ports': [], 'availableStorage': [], 'storage': [], 'memoryQuota': 271, 'memcached': 11210, 'id': 'ns_1@10.112.205.102', 'ip': '10.112.205.102', 'internal_ip': '', 'rest_username': '', 'rest_password': '', 'port': '8091', 'services': ['kv'], 'storageTotalRam': 1939, 'curr_items': 0} 2022-03-09 13:53:12,730 - root - ERROR - GET http://10.112.205.102:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2022-03-09 13:53:12,731 - root - INFO - pools/default params : memoryQuota=1039 2022-03-09 13:53:12,737 - root - INFO - --> in init_cluster...Administrator,password,8091 2022-03-09 13:53:12,738 - root - INFO - settings/web params on 10.112.205.102:8091:port=8091&username=Administrator&password=password 2022-03-09 13:53:12,778 - root - INFO - --> status:True 2022-03-09 13:53:12,781 - root - INFO - SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2022-03-09 13:53:12,886 - root - INFO - SSH Connected to 10.112.205.102 as root 2022-03-09 13:53:13,113 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:53:13,385 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:53:13,387 - root - INFO - running command.raw on 10.112.205.102: curl --silent --show-error http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2022-03-09 13:53:13,592 - root - INFO - command executed successfully with root 2022-03-09 13:53:13,593 - root - INFO - ['ok'] 2022-03-09 13:53:13,602 - root - INFO - /diag/eval status on 10.112.205.102:8091: True content: [6,6] command: cluster_compat_mode:get_compat_version(). 2022-03-09 13:53:13,606 - root - INFO - /diag/eval status on 10.112.205.102:8091: True content: [6,6] command: cluster_compat_mode:get_compat_version(). 2022-03-09 13:53:13,610 - root - INFO - settings/indexes params : storageMode=plasma 2022-03-09 13:53:13,627 - root - INFO - server: ip:10.112.205.103 port:8091 ssh_username:root, nodes/self 2022-03-09 13:53:13,632 - root - INFO - {'uptime': '13', 'memoryTotal': 2033594368, 'memoryFree': 1678716928, 'mcdMemoryReserved': 1551, 'mcdMemoryAllocated': 1551, 'status': 'healthy', 'hostname': '10.112.205.103:8091', 'clusterCompatibility': 393222, 'clusterMembership': 'active', 'recoveryType': 'none', 'version': '6.6.5-10080-enterprise', 'os': 'x86_64-unknown-linux-gnu', 'ports': [], 'availableStorage': [], 'storage': [], 'memoryQuota': 271, 'memcached': 11210, 'id': 'ns_1@10.112.205.103', 'ip': '10.112.205.103', 'internal_ip': '', 'rest_username': '', 'rest_password': '', 'port': '8091', 'services': ['kv'], 'storageTotalRam': 1939, 'curr_items': 0} 2022-03-09 13:53:13,635 - root - ERROR - GET http://10.112.205.103:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2022-03-09 13:53:13,636 - root - INFO - pools/default params : memoryQuota=1039 2022-03-09 13:53:13,642 - root - INFO - --> in init_cluster...Administrator,password,8091 2022-03-09 13:53:13,643 - root - INFO - settings/web params on 10.112.205.103:8091:port=8091&username=Administrator&password=password 2022-03-09 13:53:13,682 - root - INFO - --> status:True 2022-03-09 13:53:13,686 - root - INFO - SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2022-03-09 13:53:13,782 - root - INFO - SSH Connected to 10.112.205.103 as root 2022-03-09 13:53:14,008 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:53:14,322 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:53:14,324 - root - INFO - running command.raw on 10.112.205.103: curl --silent --show-error http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2022-03-09 13:53:14,549 - root - INFO - command executed successfully with root 2022-03-09 13:53:14,550 - root - INFO - ['ok'] 2022-03-09 13:53:14,563 - root - INFO - /diag/eval status on 10.112.205.103:8091: True content: [6,6] command: cluster_compat_mode:get_compat_version(). 2022-03-09 13:53:14,566 - root - INFO - /diag/eval status on 10.112.205.103:8091: True content: [6,6] command: cluster_compat_mode:get_compat_version(). 2022-03-09 13:53:14,572 - root - INFO - settings/indexes params : storageMode=plasma 2022-03-09 13:53:14,588 - root - INFO - server: ip:10.112.205.104 port:8091 ssh_username:root, nodes/self 2022-03-09 13:53:14,593 - root - INFO - {'uptime': '7', 'memoryTotal': 2033594368, 'memoryFree': 1663692800, 'mcdMemoryReserved': 1551, 'mcdMemoryAllocated': 1551, 'status': 'healthy', 'hostname': '10.112.205.104:8091', 'clusterCompatibility': 393222, 'clusterMembership': 'active', 'recoveryType': 'none', 'version': '6.6.5-10080-enterprise', 'os': 'x86_64-unknown-linux-gnu', 'ports': [], 'availableStorage': [], 'storage': [], 'memoryQuota': 271, 'memcached': 11210, 'id': 'ns_1@10.112.205.104', 'ip': '10.112.205.104', 'internal_ip': '', 'rest_username': '', 'rest_password': '', 'port': '8091', 'services': ['kv'], 'storageTotalRam': 1939, 'curr_items': 0} 2022-03-09 13:53:14,597 - root - ERROR - GET http://10.112.205.104:8091/pools/default body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"unknown pool"' auth: Administrator:password 2022-03-09 13:53:14,598 - root - INFO - pools/default params : memoryQuota=1039 2022-03-09 13:53:14,605 - root - INFO - --> in init_cluster...Administrator,password,8091 2022-03-09 13:53:14,606 - root - INFO - settings/web params on 10.112.205.104:8091:port=8091&username=Administrator&password=password 2022-03-09 13:53:14,653 - root - INFO - --> status:True 2022-03-09 13:53:14,656 - root - INFO - SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2022-03-09 13:53:14,796 - root - INFO - SSH Connected to 10.112.205.104 as root 2022-03-09 13:53:15,003 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:53:15,284 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:53:15,286 - root - INFO - running command.raw on 10.112.205.104: curl --silent --show-error http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2022-03-09 13:53:15,492 - root - INFO - command executed successfully with root 2022-03-09 13:53:15,492 - root - INFO - ['ok'] 2022-03-09 13:53:15,500 - root - INFO - /diag/eval status on 10.112.205.104:8091: True content: [6,6] command: cluster_compat_mode:get_compat_version(). 2022-03-09 13:53:15,503 - root - INFO - /diag/eval status on 10.112.205.104:8091: True content: [6,6] command: cluster_compat_mode:get_compat_version(). 2022-03-09 13:53:15,507 - root - INFO - settings/indexes params : storageMode=plasma 2022-03-09 13:53:15,520 - root - INFO - **** add built-in 'cbadminbucket' user to node 10.112.205.101 **** 2022-03-09 13:53:15,532 - root - ERROR - DELETE http://10.112.205.101:8091/settings/rbac/users/local/cbadminbucket body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==', 'Accept': '*/*'} error: 404 reason: unknown b'"User was not found."' auth: Administrator:password 2022-03-09 13:53:15,534 - root - INFO - Exception while deleting user. Exception is -b'"User was not found."' 2022-03-09 13:53:15,573 - root - INFO - sleep for 5 secs. ... 2022-03-09 13:53:20,578 - root - INFO - **** add 'admin' role to 'cbadminbucket' user **** 2022-03-09 13:53:20,591 - root - INFO - done initializing cluster 2022-03-09 13:53:21,542 - root - INFO - adding node 10.112.205.102:8091 to cluster 2022-03-09 13:53:21,550 - root - INFO - adding remote node @10.112.205.102:18091 to this cluster @10.112.205.101:8091 2022-03-09 13:53:31,562 - root - INFO - rebalance progress took 10.01 seconds 2022-03-09 13:53:31,563 - root - INFO - sleep for 10 seconds after rebalance... 2022-03-09 13:53:42,944 - root - INFO - adding node 10.112.205.103:8091 to cluster 2022-03-09 13:53:42,950 - root - INFO - adding remote node @10.112.205.103:18091 to this cluster @10.112.205.101:8091 2022-03-09 13:53:52,961 - root - INFO - rebalance progress took 10.01 seconds 2022-03-09 13:53:52,962 - root - INFO - sleep for 10 seconds after rebalance... 2022-03-09 13:54:04,257 - root - INFO - adding node 10.112.205.104:8091 to cluster 2022-03-09 13:54:04,264 - root - INFO - adding remote node @10.112.205.104:18091 to this cluster @10.112.205.101:8091 2022-03-09 13:54:14,277 - root - INFO - rebalance progress took 10.01 seconds 2022-03-09 13:54:14,278 - root - INFO - sleep for 10 seconds after rebalance... 2022-03-09 13:54:25,661 - root - INFO - Node 10.112.205.102 not part of cluster inactiveAdded 2022-03-09 13:54:25,663 - root - INFO - Node 10.112.205.103 not part of cluster inactiveAdded 2022-03-09 13:54:25,663 - root - INFO - Node 10.112.205.104 not part of cluster inactiveAdded 2022-03-09 13:54:25,678 - root - INFO - rebalance params : {'knownNodes': 'ns_1@10.112.205.101,ns_1@10.112.205.102,ns_1@10.112.205.103,ns_1@10.112.205.104', 'ejectedNodes': '', 'user': 'Administrator', 'password': 'password'} 2022-03-09 13:54:25,687 - root - INFO - rebalance operation started 2022-03-09 13:54:35,694 - root - INFO - Rebalance - status: none, progress: 100.00% 2022-03-09 13:54:35,700 - root - INFO - rebalancing was completed with progress: 100% in 10.011934995651245 sec 2022-03-09 13:54:35,705 - root - INFO - SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2022-03-09 13:54:35,816 - root - INFO - SSH Connected to 10.112.205.101 as root 2022-03-09 13:54:35,907 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:54:36,225 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:54:36,227 - root - INFO - running command.raw on 10.112.205.101: curl --silent --show-error http://Administrator:password@localhost:8091/diag/eval -X POST -d 'ns_config:set(allow_nonlocal_eval, true).' 2022-03-09 13:54:36,430 - root - INFO - command executed successfully with root 2022-03-09 13:54:36,431 - root - INFO - ['ok'] 2022-03-09 13:54:36,433 - root - INFO - Enabled diag/eval for non-local hosts from 10.112.205.101 2022-03-09 13:54:36,720 - root - INFO - http://10.112.205.101:8091/pools/default/buckets with param: name=standard_bucket0&ramQuotaMB=986&replicaNumber=1&bucketType=membase&replicaIndex=1&threadsNumber=3&flushEnabled=1&evictionPolicy=valueOnly&compressionMode=passive&storageBackend=couchstore 2022-03-09 13:54:36,730 - root - INFO - 0.01 seconds to create bucket standard_bucket0 2022-03-09 13:54:36,732 - root - INFO - waiting for memcached bucket : standard_bucket0 in 10.112.205.101 to accept set ops 2022-03-09 13:54:37,439 - root - INFO - creating direct client 10.112.205.101:11210 standard_bucket0 2022-03-09 13:54:37,592 - root - INFO - creating direct client 10.112.205.102:11210 standard_bucket0 2022-03-09 13:54:37,714 - root - INFO - creating direct client 10.112.205.101:11210 standard_bucket0 2022-03-09 13:54:37,787 - root - INFO - creating direct client 10.112.205.102:11210 standard_bucket0 2022-03-09 13:54:39,148 - root - INFO - bucket 'standard_bucket0' was created with per node RAM quota: 986 2022-03-09 13:54:39,149 - root - INFO - ============== basetestcase setup was finished for test #1 test_online_upgrade ============== 2022-03-09 13:54:39,160 - root - INFO - SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2022-03-09 13:54:39,215 - root - INFO - SSH Connected to 10.112.205.101 as root 2022-03-09 13:54:39,296 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:54:39,553 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:54:39,557 - root - INFO - SSH Connecting to 10.112.205.102 with username:root, attempt#1 of 5 2022-03-09 13:54:39,674 - root - INFO - SSH Connected to 10.112.205.102 as root 2022-03-09 13:54:39,771 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:54:40,062 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:54:40,067 - root - INFO - SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2022-03-09 13:54:40,130 - root - INFO - SSH Connected to 10.112.205.103 as root 2022-03-09 13:54:40,202 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:54:40,491 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:54:40,496 - root - INFO - SSH Connecting to 10.112.205.104 with username:root, attempt#1 of 5 2022-03-09 13:54:40,611 - root - INFO - SSH Connected to 10.112.205.104 as root 2022-03-09 13:54:40,724 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:54:40,973 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:54:46,108 - root - INFO - ------- Cluster statistics ------- 2022-03-09 13:54:46,109 - root - INFO - 10.112.205.101:8091 => {'services': ['kv', 'n1ql'], 'cpu_utilization': 13.68421052631579, 'mem_free': 1437941760, 'mem_total': 1930829824, 'swap_mem_used': 0, 'swap_mem_total': 1107292160} 2022-03-09 13:54:46,109 - root - INFO - 10.112.205.102:8091 => {'services': ['kv'], 'cpu_utilization': 19.10112359550562, 'mem_free': 1587126272, 'mem_total': 2033594368, 'swap_mem_used': 0, 'swap_mem_total': 1107292160} 2022-03-09 13:54:46,110 - root - INFO - 10.112.205.103:8091 => {'services': ['index'], 'cpu_utilization': 1.030927835051546, 'mem_free': 1601744896, 'mem_total': 2033594368, 'swap_mem_used': 0, 'swap_mem_total': 1107292160} 2022-03-09 13:54:46,110 - root - INFO - 10.112.205.104:8091 => {'services': ['index'], 'cpu_utilization': 0, 'mem_free': 1589276672, 'mem_total': 2033594368, 'swap_mem_used': 0, 'swap_mem_total': 1107292160} 2022-03-09 13:54:46,111 - root - INFO - --- End of cluster statistics --- 2022-03-09 13:54:46,112 - root - INFO - Time to execute basesetup : 137.2630000114441 2022-03-09 13:54:46,114 - root - INFO - SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2022-03-09 13:54:46,167 - root - INFO - SSH Connected to 10.112.205.101 as root 2022-03-09 13:54:46,249 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:54:46,539 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:54:50,101 - root - INFO - list of n1ql nodes in cluster: [ip:10.112.205.101 port:8091 ssh_username:root] 2022-03-09 13:54:50,110 - root - INFO - cannot find service node eventing in cluster 2022-03-09 13:54:50,119 - root - INFO - list of n1ql nodes in cluster: [ip:10.112.205.101 port:8091 ssh_username:root] 2022-03-09 13:54:50,125 - root - INFO - ============== EventingBaseTest setup has completed ============== 2022-03-09 13:54:50,128 - root - INFO - SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2022-03-09 13:54:50,197 - root - INFO - SSH Connected to 10.112.205.101 as root 2022-03-09 13:54:50,289 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:54:50,559 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:54:53,498 - root - INFO - list of n1ql nodes in cluster: [ip:10.112.205.101 port:8091 ssh_username:root] 2022-03-09 13:54:53,501 - root - INFO - SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2022-03-09 13:54:53,553 - root - INFO - SSH Connected to 10.112.205.101 as root 2022-03-09 13:54:53,633 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:54:53,884 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:54:53,920 - root - INFO - Initial status of 10.112.205.101 cluster is healthy 2022-03-09 13:54:53,920 - root - INFO - current status of 10.112.205.101 is healthy 2022-03-09 13:54:53,931 - root - INFO - Initial status of 10.112.205.102 cluster is healthy 2022-03-09 13:54:53,932 - root - INFO - current status of 10.112.205.102 is healthy 2022-03-09 13:54:53,943 - root - INFO - Initial status of 10.112.205.103 cluster is healthy 2022-03-09 13:54:53,943 - root - INFO - current status of 10.112.205.103 is healthy 2022-03-09 13:54:53,954 - root - INFO - Initial status of 10.112.205.104 cluster is healthy 2022-03-09 13:54:53,955 - root - INFO - current status of 10.112.205.104 is healthy 2022-03-09 13:54:53,962 - root - INFO - list of index nodes in cluster: [ip:10.112.205.103 port:8091 ssh_username:root, ip:10.112.205.104 port:8091 ssh_username:root] 2022-03-09 13:54:53,978 - root - INFO - {'indexer.settings.storage_mode': 'plasma'} set 2022-03-09 13:54:53,979 - root - INFO - Allowing the indexer to complete restart after setting the internal settings 2022-03-09 13:54:53,980 - root - INFO - sleep for 5 secs. ... 2022-03-09 13:54:58,996 - root - INFO - {'indexer.api.enableTestServer': True} set 2022-03-09 13:54:59,002 - root - INFO - SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2022-03-09 13:54:59,110 - root - INFO - SSH Connected to 10.112.205.101 as root 2022-03-09 13:54:59,235 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:54:59,591 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:55:04,664 - root - INFO - create 20160.0 to standard_bucket0 documents... 2022-03-09 13:55:04,707 - root - INFO - creating direct client 10.112.205.101:11210 standard_bucket0 2022-03-09 13:55:04,776 - root - INFO - creating direct client 10.112.205.102:11210 standard_bucket0 2022-03-09 13:55:24,637 - root - INFO - LOAD IS FINISHED 2022-03-09 13:55:24,647 - root - INFO - {'update': {'start': 0, 'end': 5}, 'remaining': {'start': 5, 'end': 10}} 2022-03-09 13:55:28,602 - root - INFO - list of n1ql nodes in cluster: [ip:10.112.205.101 port:8091 ssh_username:root] 2022-03-09 13:55:28,603 - root - INFO - ip:10.112.205.101 port:8091 ssh_username:root 2022-03-09 13:55:28,604 - root - INFO - sleep for 30 secs. ... 2022-03-09 13:55:58,610 - root - INFO - Check if index existed in standard_bucket0 on server 10.112.205.101 2022-03-09 13:55:58,616 - root - INFO - RUN QUERY SELECT * FROM system:indexes where name = '#primary' 2022-03-09 13:55:58,621 - root - INFO - query params : statement=SELECT+%2A+FROM+system%3Aindexes+where+name+%3D+%27%23primary%27 2022-03-09 13:55:58,979 - root - INFO - TOTAL ELAPSED TIME: 349.713407ms 2022-03-09 13:55:58,980 - root - ERROR - Fail to get index list. List output: {'requestID': '906b30e9-1da1-4f45-8e13-b34564db68eb', 'signature': {'*': '*'}, 'results': [], 'status': 'success', 'metrics': {'elapsedTime': '349.713407ms', 'executionTime': '319.30756ms', 'resultCount': 0, 'resultSize': 0}} 2022-03-09 13:55:58,981 - root - INFO - Create primary index 2022-03-09 13:55:58,986 - root - INFO - RUN QUERY CREATE PRIMARY INDEX ON standard_bucket0 USING GSI 2022-03-09 13:55:58,991 - root - INFO - query params : statement=CREATE+PRIMARY+INDEX+ON+standard_bucket0++USING+GSI 2022-03-09 13:56:01,943 - root - INFO - TOTAL ELAPSED TIME: 2.950242408s 2022-03-09 13:56:01,944 - root - INFO - Check if index is online 2022-03-09 13:56:01,951 - root - INFO - RUN QUERY SELECT * FROM system:indexes where name = '#primary' 2022-03-09 13:56:01,956 - root - INFO - query params : statement=SELECT+%2A+FROM+system%3Aindexes+where+name+%3D+%27%23primary%27 2022-03-09 13:56:01,977 - root - INFO - TOTAL ELAPSED TIME: 16.682934ms 2022-03-09 13:56:01,989 - root - INFO - SSH Connecting to 10.112.205.101 with username:root, attempt#1 of 5 2022-03-09 13:56:02,083 - root - INFO - SSH Connected to 10.112.205.101 as root 2022-03-09 13:56:02,166 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:56:02,416 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:56:02,438 - root - INFO - list of index nodes in cluster: [ip:10.112.205.103 port:8091 ssh_username:root, ip:10.112.205.104 port:8091 ssh_username:root] 2022-03-09 13:56:02,443 - root - INFO - Making a rest request api=http://10.112.205.103:9102/settings verb=POST params={"queryport.client.waitForScheduledIndex": false} client_cert=None verify=False 2022-03-09 13:56:02,474 - root - INFO - {'queryport.client.waitForScheduledIndex': False} set 2022-03-09 13:56:02,475 - root - INFO - Making a rest request api=http://10.112.205.103:9102/settings verb=POST params={"indexer.allowScheduleCreateRebal": true} client_cert=None verify=False 2022-03-09 13:56:02,495 - root - INFO - {'indexer.allowScheduleCreateRebal': True} set 2022-03-09 13:56:03,439 - root - INFO - RUN QUERY CREATE INDEX `index_name_0` ON standard_bucket0(job_title) USING GSI WITH {'defer_build': True} 2022-03-09 13:56:03,445 - root - INFO - query params : statement=CREATE+INDEX+%60index_name_0%60+ON+standard_bucket0%28job_title%29+USING+GSI++WITH+%7B%27defer_build%27%3A+True%7D 2022-03-09 13:56:04,029 - root - INFO - TOTAL ELAPSED TIME: 580.773663ms 2022-03-09 13:56:04,030 - root - INFO - BUILD INDEX on standard_bucket0(`index_name_0`) USING GSI 2022-03-09 13:56:05,037 - root - INFO - RUN QUERY BUILD INDEX on standard_bucket0(`index_name_0`) USING GSI 2022-03-09 13:56:05,041 - root - INFO - query params : statement=BUILD+INDEX+on+standard_bucket0%28%60index_name_0%60%29+USING+GSI 2022-03-09 13:56:05,121 - root - INFO - TOTAL ELAPSED TIME: 76.99026ms 2022-03-09 13:56:05,126 - root - INFO - RUN QUERY SELECT * FROM system:indexes where name = 'index_name_0' 2022-03-09 13:56:05,132 - root - INFO - query params : statement=SELECT+%2A+FROM+system%3Aindexes+where+name+%3D+%27index_name_0%27 2022-03-09 13:56:05,148 - root - INFO - TOTAL ELAPSED TIME: 11.113898ms 2022-03-09 13:56:06,158 - root - INFO - RUN QUERY SELECT * FROM system:indexes where name = 'index_name_0' 2022-03-09 13:56:06,166 - root - INFO - query params : statement=SELECT+%2A+FROM+system%3Aindexes+where+name+%3D+%27index_name_0%27 2022-03-09 13:56:06,194 - root - INFO - TOTAL ELAPSED TIME: 23.9984ms 2022-03-09 13:56:07,217 - root - INFO - RUN QUERY SELECT * FROM system:indexes where name = 'index_name_0' 2022-03-09 13:56:07,222 - root - INFO - query params : statement=SELECT+%2A+FROM+system%3Aindexes+where+name+%3D+%27index_name_0%27 2022-03-09 13:56:07,236 - root - INFO - TOTAL ELAPSED TIME: 11.304807ms 2022-03-09 13:56:08,243 - root - INFO - === BEFORE: ['create_index'] === 2022-03-09 13:56:08,244 - root - INFO - [ip:10.112.205.103 port:8091 ssh_username:root] 2022-03-09 13:56:09,142 - root - INFO - RUN QUERY CREATE INDEX `employeec6725e329e774019bd2596361308ac8ejob_title` ON standard_bucket0(job_title) WHERE job_title IS NOT NULL USING GSI WITH {'nodes': ['10.112.205.103:8091'], 'defer_build': True} 2022-03-09 13:56:09,147 - root - INFO - query params : statement=CREATE+INDEX+%60employeec6725e329e774019bd2596361308ac8ejob_title%60+ON+standard_bucket0%28job_title%29+WHERE++job_title+IS+NOT+NULL++USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D 2022-03-09 13:56:09,286 - root - INFO - TOTAL ELAPSED TIME: 134.557499ms 2022-03-09 13:56:09,295 - root - INFO - RUN QUERY CREATE INDEX `employeec6725e329e774019bd2596361308ac8ejoin_yr` ON standard_bucket0(join_yr) WHERE join_yr > 2010 and join_yr < 2014 USING GSI WITH {'nodes': ['10.112.205.103:8091'], 'defer_build': True} 2022-03-09 13:56:09,305 - root - INFO - query params : statement=CREATE+INDEX+%60employeec6725e329e774019bd2596361308ac8ejoin_yr%60+ON+standard_bucket0%28join_yr%29+WHERE++join_yr+%3E+2010+and+join_yr+%3C+2014++USING+GSI++WITH+%7B%27nodes%27%3A+%5B%2710.112.205.103%3A8091%27%5D%2C+%27defer_build%27%3A+True%7D 2022-03-09 13:56:09,472 - root - INFO - TOTAL ELAPSED TIME: 158.654663ms 2022-03-09 13:56:09,473 - root - INFO - BUILD INDEX on standard_bucket0(employeec6725e329e774019bd2596361308ac8ejob_title,employeec6725e329e774019bd2596361308ac8ejoin_yr) USING GSI 2022-03-09 13:56:10,482 - root - INFO - RUN QUERY BUILD INDEX on standard_bucket0(employeec6725e329e774019bd2596361308ac8ejob_title,employeec6725e329e774019bd2596361308ac8ejoin_yr) USING GSI 2022-03-09 13:56:10,487 - root - INFO - query params : statement=BUILD+INDEX+on+standard_bucket0%28employeec6725e329e774019bd2596361308ac8ejob_title%2Cemployeec6725e329e774019bd2596361308ac8ejoin_yr%29+USING+GSI 2022-03-09 13:56:10,603 - root - INFO - TOTAL ELAPSED TIME: 110.946301ms 2022-03-09 13:56:11,613 - root - INFO - RUN QUERY SELECT * FROM system:indexes where name = 'employeec6725e329e774019bd2596361308ac8ejob_title' 2022-03-09 13:56:11,621 - root - INFO - query params : statement=SELECT+%2A+FROM+system%3Aindexes+where+name+%3D+%27employeec6725e329e774019bd2596361308ac8ejob_title%27 2022-03-09 13:56:11,642 - root - INFO - TOTAL ELAPSED TIME: 17.505662ms 2022-03-09 13:56:12,648 - root - INFO - RUN QUERY SELECT * FROM system:indexes where name = 'employeec6725e329e774019bd2596361308ac8ejob_title' 2022-03-09 13:56:12,653 - root - INFO - query params : statement=SELECT+%2A+FROM+system%3Aindexes+where+name+%3D+%27employeec6725e329e774019bd2596361308ac8ejob_title%27 2022-03-09 13:56:12,669 - root - INFO - TOTAL ELAPSED TIME: 12.075309ms 2022-03-09 13:56:13,688 - root - INFO - RUN QUERY SELECT * FROM system:indexes where name = 'employeec6725e329e774019bd2596361308ac8ejob_title' 2022-03-09 13:56:13,694 - root - INFO - query params : statement=SELECT+%2A+FROM+system%3Aindexes+where+name+%3D+%27employeec6725e329e774019bd2596361308ac8ejob_title%27 2022-03-09 13:56:13,724 - root - INFO - TOTAL ELAPSED TIME: 18.975999ms 2022-03-09 13:56:13,730 - root - INFO - RUN QUERY SELECT * FROM system:indexes where name = 'employeec6725e329e774019bd2596361308ac8ejoin_yr' 2022-03-09 13:56:13,737 - root - INFO - query params : statement=SELECT+%2A+FROM+system%3Aindexes+where+name+%3D+%27employeec6725e329e774019bd2596361308ac8ejoin_yr%27 2022-03-09 13:56:13,768 - root - INFO - TOTAL ELAPSED TIME: 25.169372ms 2022-03-09 13:56:13,769 - root - INFO - === IN_BETWEEN: ['query', 'query_with_explain'] === 2022-03-09 13:56:13,771 - root - INFO - Query : SELECT * FROM standard_bucket0 where job_title IS NOT NULL ORDER BY job_title,_id 2022-03-09 13:56:13,777 - root - INFO - FROM clause ===== is standard_bucket0 where job_title IS NOT NULL 2022-03-09 13:56:13,778 - root - INFO - WHERE clause ===== is None 2022-03-09 13:56:13,780 - root - INFO - UNNEST clause ===== is None 2022-03-09 13:56:13,784 - root - INFO - SELECT clause ===== is {"*" : doc,} 2022-03-09 13:56:13,786 - root - INFO - ORDER clause ========= is doc["job_title"],doc["_id"], 2022-03-09 13:56:13,788 - root - INFO - ORDER clause ========= is doc["job_title"],doc["_id"], 2022-03-09 13:56:13,789 - root - INFO - -->select_clause:{"*" : doc,"job_title" : doc["job_title"],"_id" : doc["_id"]}; where_clause=None 2022-03-09 13:56:14,241 - root - INFO - ORDER clause ========= is doc["job_title"],doc["_id"], 2022-03-09 13:56:14,578 - root - INFO - Query : SELECT * FROM standard_bucket0 WHERE join_yr > 2010 and join_yr < 2014 ORDER BY _id 2022-03-09 13:56:14,579 - root - INFO - FROM clause ===== is standard_bucket0 2022-03-09 13:56:14,581 - root - INFO - WHERE clause ===== is doc["join_yr"] > 2010 and doc["join_yr"] < 2014 2022-03-09 13:56:14,582 - root - INFO - UNNEST clause ===== is None 2022-03-09 13:56:14,582 - root - INFO - SELECT clause ===== is {"*" : doc,} 2022-03-09 13:56:14,583 - root - INFO - ORDER clause ========= is doc["_id"], 2022-03-09 13:56:14,584 - root - INFO - ORDER clause ========= is doc["_id"], 2022-03-09 13:56:14,584 - root - INFO - -->select_clause:{"*" : doc,"_id" : doc["_id"]}; where_clause= doc["join_yr"] > 2010 and doc["join_yr"] < 2014 2022-03-09 13:56:14,585 - root - INFO - -->where_clause= doc["join_yr"] > 2010 and doc["join_yr"] < 2014 2022-03-09 13:56:14,780 - root - INFO - <<<<< START Executing Query EXPLAIN SELECT * FROM standard_bucket0 where job_title IS NOT NULL ORDER BY job_title,_id >>>>>> 2022-03-09 13:56:14,846 - root - INFO - RUN QUERY EXPLAIN SELECT * FROM standard_bucket0 where job_title IS NOT NULL ORDER BY job_title,_id 2022-03-09 13:56:14,884 - root - INFO - query params : statement=EXPLAIN+SELECT+%2A+FROM+standard_bucket0+where+job_title+IS+NOT+NULL+ORDER+BY+job_title%2C_id 2022-03-09 13:56:14,940 - root - INFO - TOTAL ELAPSED TIME: 4.43662ms 2022-03-09 13:56:14,952 - root - INFO - {'requestID': 'f9c81757-9dde-4d60-9fd1-cd66e93362e3', 'signature': 'json', 'results': [{'plan': {'#operator': 'Sequence', '~children': [{'#operator': 'Sequence', '~children': [{'#operator': 'IndexScan3', 'index': 'employeec6725e329e774019bd2596361308ac8ejob_title', 'index_id': 'e67400c50ca8aeab', 'index_projection': {'primary_key': True}, 'keyspace': 'standard_bucket0', 'namespace': 'default', 'spans': [{'exact': True, 'range': [{'inclusion': 0, 'low': 'null'}]}], 'using': 'gsi'}, {'#operator': 'Fetch', 'keyspace': 'standard_bucket0', 'namespace': 'default'}, {'#operator': 'Parallel', '~child': {'#operator': 'Sequence', '~children': [{'#operator': 'Filter', 'condition': '((`standard_bucket0`.`job_title`) is not null)'}, {'#operator': 'InitialProject', 'result_terms': [{'expr': 'self', 'star': True}]}]}}]}, {'#operator': 'Order', 'sort_terms': [{'expr': '(`standard_bucket0`.`job_title`)'}, {'expr': '(`standard_bucket0`.`_id`)'}]}, {'#operator': 'FinalProject'}]}, 'text': 'SELECT * FROM standard_bucket0 where job_title IS NOT NULL ORDER BY job_title,_id'}], 'status': 'success', 'metrics': {'elapsedTime': '4.43662ms', 'executionTime': '4.352257ms', 'resultCount': 1, 'resultSize': 919}} 2022-03-09 13:56:14,965 - root - INFO - <<<<< Done Executing Query EXPLAIN SELECT * FROM standard_bucket0 where job_title IS NOT NULL ORDER BY job_title,_id >>>>>> 2022-03-09 13:56:14,985 - root - INFO - <<<<< START Executing Query EXPLAIN SELECT * FROM standard_bucket0 WHERE join_yr > 2010 and join_yr < 2014 ORDER BY _id >>>>>> 2022-03-09 13:56:15,052 - root - INFO - RUN QUERY EXPLAIN SELECT * FROM standard_bucket0 WHERE join_yr > 2010 and join_yr < 2014 ORDER BY _id 2022-03-09 13:56:15,088 - root - INFO - ORDER clause ========= is doc["_id"], 2022-03-09 13:56:15,108 - root - INFO - query params : statement=EXPLAIN+SELECT+%2A+FROM+standard_bucket0+WHERE++join_yr+%3E+2010+and+join_yr+%3C+2014+ORDER+BY+_id+ 2022-03-09 13:56:15,178 - root - INFO - TOTAL ELAPSED TIME: 2.250184ms 2022-03-09 13:56:15,179 - root - INFO - {'requestID': 'b4f214fe-e59a-41f9-a0be-34d4c997e48b', 'signature': 'json', 'results': [{'plan': {'#operator': 'Sequence', '~children': [{'#operator': 'Sequence', '~children': [{'#operator': 'IndexScan3', 'index': 'employeec6725e329e774019bd2596361308ac8ejoin_yr', 'index_id': '56e79c9429ed21a', 'index_projection': {'primary_key': True}, 'keyspace': 'standard_bucket0', 'namespace': 'default', 'spans': [{'exact': True, 'range': [{'high': '2014', 'inclusion': 0, 'low': '2010'}]}], 'using': 'gsi'}, {'#operator': 'Fetch', 'keyspace': 'standard_bucket0', 'namespace': 'default'}, {'#operator': 'Parallel', '~child': {'#operator': 'Sequence', '~children': [{'#operator': 'Filter', 'condition': '((2010 < (`standard_bucket0`.`join_yr`)) and ((`standard_bucket0`.`join_yr`) < 2014))'}, {'#operator': 'InitialProject', 'result_terms': [{'expr': 'self', 'star': True}]}]}}]}, {'#operator': 'Order', 'sort_terms': [{'expr': '(`standard_bucket0`.`_id`)'}]}, {'#operator': 'FinalProject'}]}, 'text': 'SELECT * FROM standard_bucket0 WHERE join_yr > 2010 and join_yr < 2014 ORDER BY _id'}], 'status': 'success', 'metrics': {'elapsedTime': '2.250184ms', 'executionTime': '2.219739ms', 'resultCount': 1, 'resultSize': 948}} 2022-03-09 13:56:15,186 - root - INFO - <<<<< Done Executing Query EXPLAIN SELECT * FROM standard_bucket0 WHERE join_yr > 2010 and join_yr < 2014 ORDER BY _id >>>>>> 2022-03-09 13:56:15,193 - root - INFO - <<<<< START Executing Query SELECT * FROM standard_bucket0 where job_title IS NOT NULL ORDER BY job_title,_id >>>>>> 2022-03-09 13:56:15,251 - root - INFO - RUN QUERY SELECT * FROM standard_bucket0 where job_title IS NOT NULL ORDER BY job_title,_id 2022-03-09 13:56:15,293 - root - INFO - query params : statement=SELECT+%2A+FROM+standard_bucket0+where+job_title+IS+NOT+NULL+ORDER+BY+job_title%2C_id&scan_consistency=request_plus 2022-03-09 13:56:16,972 - root - INFO - create 20160 to standard_bucket0 documents... 2022-03-09 13:56:17,026 - root - INFO - creating direct client 10.112.205.101:11210 standard_bucket0 2022-03-09 13:56:17,107 - root - INFO - creating direct client 10.112.205.102:11210 standard_bucket0 2022-03-09 13:56:17,331 - gsi.upgrade_gsi - INFO - Upgrading servers to 7.1.0-2445... 2022-03-09 13:56:17,542 - root - INFO - rebalance params : {'knownNodes': 'ns_1@10.112.205.101,ns_1@10.112.205.102,ns_1@10.112.205.103,ns_1@10.112.205.104', 'ejectedNodes': 'ns_1@10.112.205.103', 'user': 'Administrator', 'password': 'password'} 2022-03-09 13:56:17,554 - root - INFO - rebalance operation started 2022-03-09 13:56:19,377 - root - INFO - TOTAL ELAPSED TIME: 3.633689758s 2022-03-09 13:56:19,379 - root - INFO - Analyzing Actual Result 2022-03-09 13:56:19,401 - root - INFO - Analyzing Expected Result 2022-03-09 13:56:27,621 - root - INFO - rebalance percentage : 87.00 % 2022-03-09 13:56:27,628 - root - INFO - Rebalance - status: running, progress: 87.00% 2022-03-09 13:56:47,878 - root - INFO - Rebalance - status: none, progress: 100.00% 2022-03-09 13:56:48,161 - root - INFO - rebalancing was completed with progress: 100% in 30.605792999267578 sec 2022-03-09 13:56:48,185 - root - INFO - servers ['10.112.205.103'] will be upgraded to 7.1.0-2445 version 2022-03-09 13:56:48,234 - root - INFO - SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2022-03-09 13:56:48,806 - root - INFO - SSH Connected to 10.112.205.103 as root 2022-03-09 13:56:49,656 - root - INFO - os_distro: CentOS, os_version: centos 7, is_linux_distro: True 2022-03-09 13:56:50,470 - root - INFO - extract_remote_info-->distribution_type: CentOS, distribution_version: centos 7 2022-03-09 13:56:50,480 - root - INFO - finding build 7.1.0-2445 for machine ip:10.112.205.103 port:8091 ssh_username:root 2022-03-09 13:56:50,484 - root - INFO - running command.raw on 10.112.205.103: /sbin/iptables -F 2022-03-09 13:56:50,611 - root - INFO - command executed successfully with root 2022-03-09 13:56:50,614 - root - INFO - running command.raw on 10.112.205.103: /sbin/iptables -t nat -F 2022-03-09 13:56:50,692 - root - INFO - command executed successfully with root 2022-03-09 13:56:50,699 - root - INFO - SSH Connecting to 10.112.205.103 with username:root, attempt#1 of 5 2022-03-09 13:56:51,173 - root - INFO - SSH Connected to 10.112.205.103 as root 2022-03-09 13:56:51,187 - root - INFO - running command.raw on 10.112.205.103: cd /tmp;wget -nc -q -O couchbase-server-enterprise-7.1.0-2445-centos7.x86_64.rpm http://172.23.126.166/builds/latestbuilds/couchbase-server/neo/2445/couchbase-server-enterprise-7.1.0-2445-centos7.x86_64.rpm;cd /tmp;ls -lh 2022-03-09 13:56:51,320 - root - INFO - command executed successfully with root 2022-03-09 13:56:51,322 - root - INFO - total 780M 2022-03-09 13:56:51,324 - root - INFO - -rw-r--r-- 1 root root 370M Mar 8 22:12 couchbase-server-enterprise-6.6.5-10080-centos7.x86_64.rpm 2022-03-09 13:56:51,333 - root - INFO - -rw-r--r-- 1 root root 409M Mar 8 22:12 couchbase-server-enterprise-7.1.0-2445-centos7.x86_64.rpm 2022-03-09 13:56:51,335 - root - INFO - -rw-r--r-- 1 root root 1.1M Mar 8 00:41 node3-cb660-centos7.vagrants.zip 2022-03-09 13:56:51,355 - root - INFO - drwx------ 3 root root 16 Mar 9 00:03 systemd-private-InKTAM 2022-03-09 13:56:51,385 - root - INFO - drwx------ 3 root root 16 Mar 8 23:18 systemd-private-J4zlil 2022-03-09 13:56:51,393 - root - INFO - drwx------ 2 root root 26 Mar 8 23:51 tmpmyocayvp 2022-03-09 13:56:51,408 - root - INFO - drwxrwxrwx 3 root root 55 Nov 8 01:02 vagrant-puppet 2022-03-09 13:56:51,710 - root - INFO - deliverable_type : rpm 2022-03-09 13:56:51,719 - root - INFO - /tmp/couchbase-server-enterprise-7.1.0-2445-centos7.x86_64.rpm or /tmp/ 2022-03-09 13:56:51,732 - root - INFO - running command.raw on 10.112.205.103: export CB_MASTER_PASSWORD=password; rpm -U /tmp/couchbase-server-enterprise-7.1.0-2445-centos7.x86_64.rpm 2022-03-09 13:58:43,834 - root - INFO - <<<<< Done Executing Query SELECT * FROM standard_bucket0 where job_title IS NOT NULL ORDER BY job_title,_id >>>>>> 2022-03-09 13:58:43,839 - root - INFO - <<<<< Done VERIFYING Query EXPLAIN SELECT * FROM standard_bucket0 where job_title IS NOT NULL ORDER BY job_title,_id >>>>>> 2022-03-09 13:58:43,845 - root - INFO - <<<<< Done VERIFYING Query EXPLAIN SELECT * FROM standard_bucket0 WHERE join_yr > 2010 and join_yr < 2014 ORDER BY _id >>>>>> 2022-03-09 13:58:43,852 - root - INFO - <<<<< Done VERIFYING Query SELECT * FROM standard_bucket0 where job_title IS NOT NULL ORDER BY job_title,_id >>>>>> 2022-03-09 13:58:43,855 - root - INFO - <<<<< START Executing Query SELECT * FROM standard_bucket0 WHERE join_yr > 2010 and join_yr < 2014 ORDER BY _id >>>>>> 2022-03-09 13:58:43,878 - root - INFO - RUN QUERY SELECT * FROM standard_bucket0 WHERE join_yr > 2010 and join_yr < 2014 ORDER BY _id 2022-03-09 13:58:43,900 - root - INFO - query params : statement=SELECT+%2A+FROM+standard_bucket0+WHERE++join_yr+%3E+2010+and+join_yr+%3C+2014+ORDER+BY+_id+&scan_consistency=request_plus 2022-03-09 13:58:54,235 - root - INFO - TOTAL ELAPSED TIME: 10.130839851s 2022-03-09 13:58:54,242 - root - INFO - Analyzing Actual Result 2022-03-09 13:58:54,270 - root - INFO - Analyzing Expected Result 2022-03-09 13:59:09,142 - root - INFO - command executed successfully with root 2022-03-09 13:59:09,158 - root - INFO - Found 'You have successfully installed Couchbase Server.' in output 2022-03-09 13:59:09,178 - root - INFO - running command.raw on 10.112.205.103: sed -i 's/export PATH/export PATH\nexport CBFT_ENV_OPTIONS=bleveMaxResultWindow=10000000/' /opt/couchbase/bin/couchbase-server 2022-03-09 13:59:09,296 - root - INFO - command executed successfully with root 2022-03-09 13:59:09,303 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:59:09,499 - root - INFO - process beam.smp is running on 10.112.205.103: with pid 9127 2022-03-09 13:59:10,270 - root - INFO - Running systemd command on this server 2022-03-09 13:59:10,279 - root - INFO - running command.raw on 10.112.205.103: systemctl stop couchbase-server.service 2022-03-09 13:59:10,414 - root - INFO - command executed successfully with root 2022-03-09 13:59:10,427 - root - INFO - Checking for process beam.smp on linux 2022-03-09 13:59:10,611 - root - INFO - process beam.smp is running on 10.112.205.103: with pid 9127 2022-03-09 13:59:10,613 - root - INFO - upgrade 10.112.205.103 to version 7.1.0-2445 is completed 2022-03-09 13:59:10,908 - root - INFO - 10.112.205.103 **** The linux version file /opt/couchbase/ VERSION.txt exists 2022-03-09 13:59:11,676 - root - INFO - Running systemd start command on this server 2022-03-09 13:59:11,678 - root - INFO - running command.raw on 10.112.205.103: systemctl start couchbase-server.service 2022-03-09 13:59:11,866 - root - INFO - command executed successfully with root 2022-03-09 13:59:11,910 - root - INFO - sleep for 30 secs. wait for servers up ... 2022-03-09 13:59:42,074 - root - INFO - waiting for ns_server @ 10.112.205.103:8091 2022-03-09 13:59:42,088 - root - INFO - -->is_ns_server_running? 2022-03-09 13:59:42,155 - root - INFO - ns_server @ 10.112.205.103:8091 is running 2022-03-09 13:59:42,163 - root - INFO - --> in init_cluster...Administrator,password,8091 2022-03-09 13:59:42,176 - root - INFO - settings/web params on 10.112.205.103:8091:port=8091&username=Administrator&password=password 2022-03-09 13:59:42,439 - root - INFO - --> status:True 2022-03-09 13:59:42,441 - root - INFO - sleep for 15 secs. ... 2022-03-09 13:59:57,468 - root - INFO - sleep for 10 secs. ... 2022-03-09 14:00:07,502 - gsi.upgrade_gsi - INFO - ==== Upgrade Complete ==== 2022-03-09 14:00:07,503 - root - INFO - sleep for 120 secs. ... 2022-03-09 14:00:11,022 - root - INFO - <<<<< Done Executing Query SELECT * FROM standard_bucket0 WHERE join_yr > 2010 and join_yr < 2014 ORDER BY _id >>>>>> 2022-03-09 14:00:11,023 - root - INFO - <<<<< Done VERIFYING Query SELECT * FROM standard_bucket0 WHERE join_yr > 2010 and join_yr < 2014 ORDER BY _id >>>>>> 2022-03-09 14:02:07,520 - root - INFO - Node versions in cluster ['7.1.0-2445-enterprise'] 2022-03-09 14:02:12,249 - root - INFO - adding node 10.112.205.103:8091 to cluster 2022-03-09 14:02:12,260 - root - INFO - adding remote node @10.112.205.103:18091 to this cluster @10.112.205.101:8091 2022-03-09 14:02:22,272 - root - INFO - rebalance progress took 10.01 seconds 2022-03-09 14:02:22,273 - root - INFO - sleep for 10 seconds after rebalance... 2022-03-09 14:02:36,389 - root - INFO - Node 10.112.205.103 not part of cluster inactiveAdded 2022-03-09 14:02:36,446 - root - INFO - rebalance params : {'knownNodes': 'ns_1@10.112.205.101,ns_1@10.112.205.102,ns_1@10.112.205.103,ns_1@10.112.205.104', 'ejectedNodes': '', 'user': 'Administrator', 'password': 'password'} 2022-03-09 14:02:46,392 - root - INFO - rebalance operation started 2022-03-09 14:02:56,400 - root - INFO - rebalance percentage : 37.00 % 2022-03-09 14:02:56,401 - root - INFO - Rebalance - status: running, progress: 37.00% 2022-03-09 14:03:16,453 - root - INFO - rebalance percentage : 37.00 % 2022-03-09 14:03:16,454 - root - INFO - Rebalance - status: running, progress: 37.00% 2022-03-09 14:03:36,514 - root - INFO - rebalance percentage : 37.00 % 2022-03-09 14:03:36,515 - root - INFO - Rebalance - status: running, progress: 37.00% 2022-03-09 14:03:56,571 - root - ERROR - {'status': 'none', 'errorMessage': 'Rebalance failed. See logs for detailed reason. You can try again.'} - rebalance failed 2022-03-09 14:03:56,578 - root - INFO - Latest logs from UI on 10.112.205.101: 2022-03-09 14:03:56,579 - root - ERROR - {'node': 'ns_1@10.112.205.102', 'type': 'info', 'code': 4, 'module': 'ns_node_disco', 'tstamp': 1646814753216, 'shortText': 'node up', 'text': "Node 'ns_1@10.112.205.102' saw that node 'ns_1@10.112.205.103' came up. Tags: []", 'serverTime': '2022-03-09T00:32:33.216Z'} 2022-03-09 14:03:56,579 - root - ERROR - {'node': 'ns_1@10.112.205.104', 'type': 'info', 'code': 4, 'module': 'ns_node_disco', 'tstamp': 1646814752897, 'shortText': 'node up', 'text': "Node 'ns_1@10.112.205.104' saw that node 'ns_1@10.112.205.103' came up. Tags: []", 'serverTime': '2022-03-09T00:32:32.897Z'} 2022-03-09 14:03:56,580 - root - ERROR - {'node': 'ns_1@10.112.205.101', 'type': 'info', 'code': 4, 'module': 'ns_node_disco', 'tstamp': 1646814752405, 'shortText': 'node up', 'text': "Node 'ns_1@10.112.205.101' saw that node 'ns_1@10.112.205.103' came up. Tags: []", 'serverTime': '2022-03-09T00:32:32.405Z'} 2022-03-09 14:03:56,581 - root - ERROR - {'node': 'ns_1@10.112.205.101', 'type': 'info', 'code': 0, 'module': 'ns_cluster', 'tstamp': 1646814752103, 'shortText': 'message', 'text': "Started node add transaction by adding node 'ns_1@10.112.205.103' to nodes_wanted (group: undefined)", 'serverTime': '2022-03-09T00:32:32.103Z'} 2022-03-09 14:03:56,581 - root - ERROR - {'node': 'ns_1@10.112.205.102', 'type': 'warning', 'code': 5, 'module': 'ns_node_disco', 'tstamp': 1646814389466, 'shortText': 'node down', 'text': "Node 'ns_1@10.112.205.102' saw that node 'ns_1@10.112.205.103' went down. Details: [{nodedown_reason,\n connection_closed}]", 'serverTime': '2022-03-09T00:26:29.466Z'} 2022-03-09 14:03:56,582 - root - ERROR - {'node': 'ns_1@10.112.205.104', 'type': 'warning', 'code': 5, 'module': 'ns_node_disco', 'tstamp': 1646814388800, 'shortText': 'node down', 'text': "Node 'ns_1@10.112.205.104' saw that node 'ns_1@10.112.205.103' went down. Details: [{nodedown_reason,\n connection_closed}]", 'serverTime': '2022-03-09T00:26:28.800Z'} 2022-03-09 14:03:56,583 - root - ERROR - {'node': 'ns_1@10.112.205.101', 'type': 'warning', 'code': 5, 'module': 'ns_node_disco', 'tstamp': 1646814388556, 'shortText': 'node down', 'text': "Node 'ns_1@10.112.205.101' saw that node 'ns_1@10.112.205.103' went down. Details: [{nodedown_reason,\n connection_closed}]", 'serverTime': '2022-03-09T00:26:28.556Z'} 2022-03-09 14:03:56,584 - root - ERROR - {'node': 'ns_1@10.112.205.103', 'type': 'info', 'code': 1, 'module': 'ns_cluster', 'tstamp': 1646814388410, 'shortText': 'message', 'text': "Node 'ns_1@10.112.205.103' is leaving cluster.", 'serverTime': '2022-03-09T00:26:28.410Z'} 2022-03-09 14:03:56,584 - root - ERROR - {'node': 'ns_1@10.112.205.101', 'type': 'info', 'code': 0, 'module': 'ns_orchestrator', 'tstamp': 1646814388367, 'shortText': 'message', 'text': 'Rebalance completed successfully.\nRebalance Operation Id = 36f9a38013ea542263ab6c253c3713d4', 'serverTime': '2022-03-09T00:26:28.367Z'} 2022-03-09 14:03:56,585 - root - ERROR - {'node': 'ns_1@10.112.205.101', 'type': 'info', 'code': 0, 'module': 'ns_vbucket_mover', 'tstamp': 1646814377780, 'shortText': 'message', 'text': 'Bucket "standard_bucket0" rebalance appears to be swap rebalance', 'serverTime': '2022-03-09T00:26:17.780Z'} [, , , , , , , , , ][, , , , , , ] Wed Mar 9 14:03:56 2022