[2018-02-28 08:56:23,918] - [rest_client] [140031333738240] - INFO - /diag/eval status on 172.23.105.81:8091: True content: "/opt/couchbase/var/lib/couchbase/logs" command: filename:absname(element(2, application:get_env(ns_server,error_logger_mf_dir))). [2018-02-28 08:56:23,921] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.81 with username:root password:couchbase ssh_key: [2018-02-28 08:56:24,093] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.81 [2018-02-28 08:56:24,491] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.81: zgrep "panic:" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:24,601] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:24,602] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:24,719] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.81 with username:root password:couchbase ssh_key: [2018-02-28 08:56:24,855] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.81 [2018-02-28 08:56:25,288] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.81: zgrep "non-recoverable error from xmem client. response status=KEY_ENOENT" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:25,392] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:25,394] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:25,398] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.86 with username:root password:couchbase ssh_key: [2018-02-28 08:56:25,602] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.86 [2018-02-28 08:56:25,880] - [remote_util] [140031333738240] - INFO - It could be other operating system. Go to check at other location [2018-02-28 08:56:26,308] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.86: zgrep "panic:" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:26,446] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:26,447] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:26,566] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.86 with username:root password:couchbase ssh_key: [2018-02-28 08:56:26,867] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.86 [2018-02-28 08:56:27,185] - [remote_util] [140031333738240] - INFO - It could be other operating system. Go to check at other location [2018-02-28 08:56:27,595] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.86: zgrep "non-recoverable error from xmem client. response status=KEY_ENOENT" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:27,716] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:27,717] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:27,721] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.9 with username:root password:couchbase ssh_key: [2018-02-28 08:56:27,862] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.9 [2018-02-28 08:56:28,333] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.9: zgrep "panic:" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:28,424] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:28,424] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:28,428] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.9 with username:root password:couchbase ssh_key: [2018-02-28 08:56:28,555] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.9 [2018-02-28 08:56:29,014] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.9: zgrep "non-recoverable error from xmem client. response status=KEY_ENOENT" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:29,104] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:29,104] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:29,109] - [remote_util] [140031333738240] - INFO - connecting to 172.23.107.67 with username:root password:couchbase ssh_key: [2018-02-28 08:56:29,286] - [remote_util] [140031333738240] - INFO - Connected to 172.23.107.67 [2018-02-28 08:56:29,713] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.67: zgrep "panic:" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:29,804] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:29,805] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:29,810] - [remote_util] [140031333738240] - INFO - connecting to 172.23.107.67 with username:root password:couchbase ssh_key: [2018-02-28 08:56:29,951] - [remote_util] [140031333738240] - INFO - Connected to 172.23.107.67 [2018-02-28 08:56:30,404] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.67: zgrep "non-recoverable error from xmem client. response status=KEY_ENOENT" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:30,493] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:30,494] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:30,611] - [remote_util] [140031333738240] - INFO - connecting to 172.23.107.78 with username:root password:couchbase ssh_key: [2018-02-28 08:56:30,751] - [remote_util] [140031333738240] - INFO - Connected to 172.23.107.78 [2018-02-28 08:56:31,155] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.78: zgrep "panic:" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:31,241] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:31,241] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:31,359] - [remote_util] [140031333738240] - INFO - connecting to 172.23.107.78 with username:root password:couchbase ssh_key: [2018-02-28 08:56:31,479] - [remote_util] [140031333738240] - INFO - Connected to 172.23.107.78 [2018-02-28 08:56:31,893] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.78: zgrep "non-recoverable error from xmem client. response status=KEY_ENOENT" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:31,978] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:31,979] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:32,097] - [remote_util] [140031333738240] - INFO - connecting to 172.23.108.109 with username:root password:couchbase ssh_key: [2018-02-28 08:56:32,301] - [remote_util] [140031333738240] - INFO - Connected to 172.23.108.109 [2018-02-28 08:56:32,954] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.108.109: zgrep "panic:" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:33,018] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:33,018] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:33,136] - [remote_util] [140031333738240] - INFO - connecting to 172.23.108.109 with username:root password:couchbase ssh_key: [2018-02-28 08:56:33,340] - [remote_util] [140031333738240] - INFO - Connected to 172.23.108.109 [2018-02-28 08:56:33,952] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.108.109: zgrep "non-recoverable error from xmem client. response status=KEY_ENOENT" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:34,011] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:34,013] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:34,128] - [xdcrnewbasetests] [140031333738240] - INFO - {'172.23.107.67': {'non-recoverable error from xmem client. response status=KEY_ENOENT': 0, 'panic:': 0}, '172.23.105.81': {'non-recoverable error from xmem client. response status=KEY_ENOENT': 0, 'panic:': 0}, '172.23.105.86': {'non-recoverable error from xmem client. response status=KEY_ENOENT': 0, 'panic:': 0}, '172.23.108.109': {'non-recoverable error from xmem client. response status=KEY_ENOENT': 0, 'panic:': 0}, '172.23.107.78': {'non-recoverable error from xmem client. response status=KEY_ENOENT': 0, 'panic:': 0}, '172.23.105.9': {'non-recoverable error from xmem client. response status=KEY_ENOENT': 0, 'panic:': 0}} [2018-02-28 08:56:34,218] - [rest_client] [140031333738240] - INFO - /diag/eval status on 172.23.105.81:8091: True content: "/opt/couchbase/var/lib/couchbase/logs" command: filename:absname(element(2, application:get_env(ns_server,error_logger_mf_dir))). [2018-02-28 08:56:34,221] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.81 with username:root password:couchbase ssh_key: [2018-02-28 08:56:34,384] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.81 [2018-02-28 08:56:34,843] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.81: zgrep "Try to fix Pipeline" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:34,951] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:34,952] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:35,069] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.86 with username:root password:couchbase ssh_key: [2018-02-28 08:56:35,247] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.86 [2018-02-28 08:56:35,537] - [remote_util] [140031333738240] - INFO - It could be other operating system. Go to check at other location [2018-02-28 08:56:35,948] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.86: zgrep "Try to fix Pipeline" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:36,075] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:36,076] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:36,080] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.9 with username:root password:couchbase ssh_key: [2018-02-28 08:56:36,224] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.9 [2018-02-28 08:56:36,630] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.9: zgrep "Try to fix Pipeline" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:36,721] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:36,722] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:36,840] - [remote_util] [140031333738240] - INFO - connecting to 172.23.107.67 with username:root password:couchbase ssh_key: [2018-02-28 08:56:37,172] - [remote_util] [140031333738240] - INFO - Connected to 172.23.107.67 [2018-02-28 08:56:37,676] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.67: zgrep "Try to fix Pipeline" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:37,767] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:37,767] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:37,772] - [remote_util] [140031333738240] - INFO - connecting to 172.23.107.78 with username:root password:couchbase ssh_key: [2018-02-28 08:56:37,881] - [remote_util] [140031333738240] - INFO - Connected to 172.23.107.78 [2018-02-28 08:56:38,376] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.78: zgrep "Try to fix Pipeline" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:38,462] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:38,464] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:38,582] - [remote_util] [140031333738240] - INFO - connecting to 172.23.108.109 with username:root password:couchbase ssh_key: [2018-02-28 08:56:38,783] - [remote_util] [140031333738240] - INFO - Connected to 172.23.108.109 [2018-02-28 08:56:39,386] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.108.109: zgrep "Try to fix Pipeline" "/opt/couchbase/var/lib/couchbase/logs"/goxdcr.log* | wc -l [2018-02-28 08:56:39,503] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:56:39,504] - [xdcrnewbasetests] [140031333738240] - INFO - 0 [2018-02-28 08:56:39,505] - [xdcrnewbasetests] [140031333738240] - INFO - {'172.23.107.67': 0, '172.23.105.81': 0, '172.23.105.86': 0, '172.23.108.109': 0, '172.23.107.78': 0, '172.23.105.9': 0} [2018-02-28 08:56:39,506] - [xdcrnewbasetests] [140031333738240] - INFO - ==== XDCRNewbasetests setup is started for test #10 test_compression_with_unixdcr_incr_load ==== [2018-02-28 08:56:39,509] - [xdcrnewbasetests] [140031333738240] - INFO - removing xdcr/nodes settings [2018-02-28 08:56:39,791] - [xdcrnewbasetests] [140031333738240] - INFO - cleanup [ip:172.23.105.81 port:8091 ssh_username:root, ip:172.23.105.86 port:8091 ssh_username:root] [2018-02-28 08:56:39,917] - [bucket_helper] [140031333738240] - INFO - deleting existing buckets [] on 172.23.105.81 [2018-02-28 08:56:39,918] - [bucket_helper] [140031333738240] - INFO - sleep 2 seconds to make sure all buckets were deleted completely. [2018-02-28 08:56:42,309] - [cluster_helper] [140031333738240] - INFO - waiting for ns_server @ 172.23.105.81:8091 [2018-02-28 08:56:42,347] - [cluster_helper] [140031333738240] - INFO - ns_server @ 172.23.105.81:8091 is running [2018-02-28 08:56:42,454] - [bucket_helper] [140031333738240] - INFO - deleting existing buckets [] on 172.23.105.86 [2018-02-28 08:56:42,455] - [bucket_helper] [140031333738240] - INFO - sleep 2 seconds to make sure all buckets were deleted completely. [2018-02-28 08:56:44,689] - [cluster_helper] [140031333738240] - INFO - waiting for ns_server @ 172.23.105.86:8091 [2018-02-28 08:56:44,726] - [cluster_helper] [140031333738240] - INFO - ns_server @ 172.23.105.86:8091 is running [2018-02-28 08:56:44,727] - [xdcrnewbasetests] [140031333738240] - INFO - removing xdcr/nodes settings [2018-02-28 08:56:44,950] - [xdcrnewbasetests] [140031333738240] - INFO - cleanup [ip:172.23.105.9 port:8091 ssh_username:root, ip:172.23.107.67 port:8091 ssh_username:root] [2018-02-28 08:56:45,018] - [bucket_helper] [140031333738240] - INFO - deleting existing buckets [] on 172.23.105.9 [2018-02-28 08:56:45,018] - [bucket_helper] [140031333738240] - INFO - sleep 2 seconds to make sure all buckets were deleted completely. [2018-02-28 08:56:47,449] - [cluster_helper] [140031333738240] - INFO - waiting for ns_server @ 172.23.105.9:8091 [2018-02-28 08:56:47,489] - [cluster_helper] [140031333738240] - INFO - ns_server @ 172.23.105.9:8091 is running [2018-02-28 08:56:47,567] - [bucket_helper] [140031333738240] - INFO - deleting existing buckets [] on 172.23.107.67 [2018-02-28 08:56:47,568] - [bucket_helper] [140031333738240] - INFO - sleep 2 seconds to make sure all buckets were deleted completely. [2018-02-28 08:56:49,747] - [cluster_helper] [140031333738240] - INFO - waiting for ns_server @ 172.23.107.67:8091 [2018-02-28 08:56:49,782] - [cluster_helper] [140031333738240] - INFO - ns_server @ 172.23.107.67:8091 is running [2018-02-28 08:56:49,782] - [xdcrnewbasetests] [140031333738240] - INFO - Initializing all clusters... [2018-02-28 08:56:49,862] - [rest_client] [140031333738240] - INFO - /diag/eval status on 172.23.105.81:8091: True content: [5,5] command: cluster_compat_mode:get_compat_version(). [2018-02-28 08:56:50,571] - [task] [140030563579648] - INFO - server: ip:172.23.105.81 port:8091 ssh_username:root, nodes/self: {'ip': u'172.23.105.81', 'availableStorage': [], 'rest_username': '', 'id': u'ns_1@172.23.105.81', 'uptime': u'4674', 'mcdMemoryReserved': 3104, 'storageTotalRam': 3880, 'hostname': u'172.23.105.81:8091', 'storage': [], 'moxi': 11211, 'port': u'8091', 'version': u'5.5.0-2006-enterprise', 'memcached': 11210, 'status': u'healthy', 'clusterCompatibility': 327685, 'curr_items': 0, 'services': [u'kv'], 'rest_password': '', 'clusterMembership': u'active', 'memoryFree': 2955739136, 'memoryTotal': 4069212160, 'memoryQuota': 2069, 'mcdMemoryAllocated': 3104, 'os': u'x86_64-unknown-linux-gnu', 'ports': []} [2018-02-28 08:56:50,572] - [rest_client] [140030563579648] - INFO - pools/default params : memoryQuota=2069 [2018-02-28 08:56:50,613] - [rest_client] [140030563579648] - INFO - settings/indexes params : storageMode=forestdb [2018-02-28 08:56:50,647] - [rest_client] [140030563579648] - ERROR - POST http://172.23.105.81:8091/settings/indexes body: storageMode=forestdb headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 400 reason: unknown {"errors":{"storageMode":"The value must be one of the following: [plasma,memory_optimized]"}} auth: Administrator:password [2018-02-28 08:56:50,647] - [rest_client] [140030563579648] - INFO - settings/web params on 172.23.105.81:8091:username=Administrator&password=password&port=8091 [2018-02-28 08:56:50,956] - [task] [140030563579648] - INFO - server: ip:172.23.105.86 port:8091 ssh_username:root, nodes/self: {'ip': u'172.23.105.86', 'availableStorage': [], 'rest_username': '', 'id': u'ns_1@172.23.105.86', 'uptime': u'4651', 'mcdMemoryReserved': 3022, 'storageTotalRam': 3778, 'hostname': u'172.23.105.86:8091', 'storage': [], 'moxi': 11211, 'port': u'8091', 'version': u'5.5.0-2006-enterprise', 'memcached': 11210, 'status': u'healthy', 'clusterCompatibility': 327685, 'curr_items': 0, 'services': [u'kv'], 'rest_password': '', 'clusterMembership': u'active', 'memoryFree': 3281260544, 'memoryTotal': 3961716736, 'memoryQuota': 2266, 'mcdMemoryAllocated': 3022, 'os': u'x86_64-unknown-linux-gnu', 'ports': []} [2018-02-28 08:56:50,956] - [rest_client] [140030563579648] - INFO - pools/default params : memoryQuota=2014 [2018-02-28 08:56:50,991] - [rest_client] [140030563579648] - INFO - settings/indexes params : storageMode=forestdb [2018-02-28 08:56:51,030] - [rest_client] [140030563579648] - ERROR - POST http://172.23.105.86:8091/settings/indexes body: storageMode=forestdb headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 400 reason: unknown {"errors":{"storageMode":"The value must be one of the following: [plasma,memory_optimized]"}} auth: Administrator:password [2018-02-28 08:56:51,031] - [rest_client] [140030563579648] - INFO - settings/web params on 172.23.105.86:8091:username=Administrator&password=password&port=8091 [2018-02-28 08:56:52,230] - [task] [140030563579648] - INFO - adding node 172.23.105.86:8091 to cluster [2018-02-28 08:56:52,230] - [rest_client] [140030563579648] - INFO - adding remote node @172.23.105.86:8091 to this cluster @172.23.105.81:8091 [2018-02-28 08:56:56,900] - [rest_client] [140030563579648] - INFO - rebalance params : {'password': 'password', 'ejectedNodes': '', 'user': 'Administrator', 'knownNodes': u'ns_1@172.23.105.81,ns_1@172.23.105.86'} [2018-02-28 08:56:56,943] - [rest_client] [140030563579648] - INFO - rebalance operation started [2018-02-28 08:56:56,995] - [rest_client] [140030563579648] - INFO - rebalance percentage : 0.00 % [2018-02-28 08:56:56,995] - [task] [140030563579648] - INFO - Rebalance - status: running, progress: 0 [2018-02-28 08:57:07,109] - [task] [140030563579648] - INFO - Rebalance - status: none, progress: 100 [2018-02-28 08:57:07,191] - [task] [140030563579648] - INFO - rebalancing was completed with progress: 100% in 10.2474730015 sec [2018-02-28 08:57:07,275] - [rest_client] [140031333738240] - INFO - /diag/eval status on 172.23.105.9:8091: True content: [5,5] command: cluster_compat_mode:get_compat_version(). [2018-02-28 08:57:07,635] - [task] [140030546794240] - INFO - server: ip:172.23.105.9 port:8091 ssh_username:root, nodes/self: {'ip': u'172.23.105.9', 'availableStorage': [], 'rest_username': '', 'id': u'ns_1@172.23.105.9', 'uptime': u'4690', 'mcdMemoryReserved': 3104, 'storageTotalRam': 3880, 'hostname': u'172.23.105.9:8091', 'storage': [], 'moxi': 11211, 'port': u'8091', 'version': u'5.5.0-2006-enterprise', 'memcached': 11210, 'status': u'healthy', 'clusterCompatibility': 327685, 'curr_items': 0, 'services': [u'kv'], 'rest_password': '', 'clusterMembership': u'active', 'memoryFree': 3249426432, 'memoryTotal': 4069322752, 'memoryQuota': 2069, 'mcdMemoryAllocated': 3104, 'os': u'x86_64-unknown-linux-gnu', 'ports': []} [2018-02-28 08:57:07,636] - [rest_client] [140030546794240] - INFO - pools/default params : memoryQuota=2069 [2018-02-28 08:57:07,678] - [rest_client] [140030546794240] - INFO - settings/indexes params : storageMode=forestdb [2018-02-28 08:57:07,716] - [rest_client] [140030546794240] - ERROR - POST http://172.23.105.9:8091/settings/indexes body: storageMode=forestdb headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 400 reason: unknown {"errors":{"storageMode":"The value must be one of the following: [plasma,memory_optimized]"}} auth: Administrator:password [2018-02-28 08:57:07,717] - [rest_client] [140030546794240] - INFO - settings/web params on 172.23.105.9:8091:username=Administrator&password=password&port=8091 [2018-02-28 08:57:08,237] - [task] [140030546794240] - INFO - server: ip:172.23.107.67 port:8091 ssh_username:root, nodes/self: {'ip': u'172.23.107.67', 'availableStorage': [], 'rest_username': '', 'id': u'ns_1@172.23.107.67', 'uptime': u'4697', 'mcdMemoryReserved': 3104, 'storageTotalRam': 3880, 'hostname': u'172.23.107.67:8091', 'storage': [], 'moxi': 11211, 'port': u'8091', 'version': u'5.5.0-2006-enterprise', 'memcached': 11210, 'status': u'healthy', 'clusterCompatibility': 327685, 'curr_items': 0, 'services': [u'kv'], 'rest_password': '', 'clusterMembership': u'active', 'memoryFree': 3440685056, 'memoryTotal': 4069322752, 'memoryQuota': 2328, 'mcdMemoryAllocated': 3104, 'os': u'x86_64-unknown-linux-gnu', 'ports': []} [2018-02-28 08:57:08,238] - [rest_client] [140030546794240] - INFO - pools/default params : memoryQuota=2069 [2018-02-28 08:57:08,270] - [rest_client] [140030546794240] - INFO - settings/indexes params : storageMode=forestdb [2018-02-28 08:57:08,304] - [rest_client] [140030546794240] - ERROR - POST http://172.23.107.67:8091/settings/indexes body: storageMode=forestdb headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 400 reason: unknown {"errors":{"storageMode":"The value must be one of the following: [plasma,memory_optimized]"}} auth: Administrator:password [2018-02-28 08:57:08,305] - [rest_client] [140030546794240] - INFO - settings/web params on 172.23.107.67:8091:username=Administrator&password=password&port=8091 [2018-02-28 08:57:09,524] - [task] [140030546794240] - INFO - adding node 172.23.107.67:8091 to cluster [2018-02-28 08:57:09,525] - [rest_client] [140030546794240] - INFO - adding remote node @172.23.107.67:8091 to this cluster @172.23.105.9:8091 [2018-02-28 08:57:11,880] - [rest_client] [140030546794240] - INFO - rebalance params : {'password': 'password', 'ejectedNodes': '', 'user': 'Administrator', 'knownNodes': u'ns_1@172.23.105.9,ns_1@172.23.107.67'} [2018-02-28 08:57:11,919] - [rest_client] [140030546794240] - INFO - rebalance operation started [2018-02-28 08:57:11,952] - [rest_client] [140030546794240] - INFO - rebalance percentage : 0.00 % [2018-02-28 08:57:11,952] - [task] [140030546794240] - INFO - Rebalance - status: running, progress: 0 [2018-02-28 08:57:22,047] - [task] [140030546794240] - INFO - Rebalance - status: none, progress: 100 [2018-02-28 08:57:22,097] - [task] [140030546794240] - INFO - rebalancing was completed with progress: 100% in 10.1768641472 sec [2018-02-28 08:57:22,325] - [rest_client] [140031333738240] - ERROR - DELETE http://172.23.105.81:8091/settings/rbac/users/local/cbadminbucket body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 404 reason: unknown "User was not found." auth: Administrator:password [2018-02-28 08:57:22,327] - [internal_user] [140031333738240] - INFO - Exception while deleting user. Exception is -"User was not found." [2018-02-28 08:57:42,815] - [rest_client] [140031333738240] - ERROR - DELETE http://172.23.105.9:8091/settings/rbac/users/local/cbadminbucket body: headers: {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*', 'Authorization': 'Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==\n'} error: 404 reason: unknown "User was not found." auth: Administrator:password [2018-02-28 08:57:42,816] - [internal_user] [140031333738240] - INFO - Exception while deleting user. Exception is -"User was not found." [2018-02-28 08:58:03,403] - [rest_client] [140030563579648] - INFO - Node versions in cluster [u'5.5.0-2006-enterprise', u'5.5.0-2006-enterprise'] [2018-02-28 08:58:03,404] - [rest_client] [140030563579648] - INFO - http://172.23.105.81:8091/pools/default/buckets with param: bucketType=membase&threadsNumber=3&authType=none&replicaIndex=1&name=standard_bucket_1&evictionPolicy=valueOnly&flushEnabled=1&replicaNumber=1&ramQuotaMB=757 [2018-02-28 08:58:03,441] - [rest_client] [140030563579648] - INFO - 0.04 seconds to create bucket standard_bucket_1 [2018-02-28 08:58:03,656] - [rest_client] [140030563579648] - INFO - Node versions in cluster [u'5.5.0-2006-enterprise', u'5.5.0-2006-enterprise'] [2018-02-28 08:58:03,656] - [rest_client] [140030563579648] - INFO - http://172.23.105.81:8091/pools/default/buckets with param: bucketType=membase&threadsNumber=3&authType=none&replicaIndex=1&name=standard_bucket_2&evictionPolicy=valueOnly&flushEnabled=1&replicaNumber=1&ramQuotaMB=757 [2018-02-28 08:58:04,487] - [rest_client] [140030563579648] - INFO - 0.83 seconds to create bucket standard_bucket_2 [2018-02-28 08:58:04,487] - [bucket_helper] [140030563579648] - INFO - waiting for memcached bucket : standard_bucket_1 in 172.23.105.81 to accept set ops [2018-02-28 08:58:04,900] - [data_helper] [140030563579648] - INFO - creating direct client 172.23.105.81:11210 standard_bucket_1 [2018-02-28 08:58:05,508] - [data_helper] [140030563579648] - INFO - creating direct client 172.23.105.86:11210 standard_bucket_1 [2018-02-28 08:58:05,908] - [data_helper] [140030563579648] - INFO - creating direct client 172.23.105.81:11210 standard_bucket_1 [2018-02-28 08:58:06,245] - [data_helper] [140030563579648] - INFO - creating direct client 172.23.105.86:11210 standard_bucket_1 [2018-02-28 08:58:07,099] - [task] [140030563579648] - INFO - bucket 'standard_bucket_1' was created with per node RAM quota: 757 [2018-02-28 08:58:07,099] - [bucket_helper] [140030563579648] - INFO - waiting for memcached bucket : standard_bucket_2 in 172.23.105.81 to accept set ops [2018-02-28 08:58:07,502] - [data_helper] [140030563579648] - INFO - creating direct client 172.23.105.81:11210 standard_bucket_2 [2018-02-28 08:58:07,859] - [data_helper] [140030563579648] - INFO - creating direct client 172.23.105.86:11210 standard_bucket_2 [2018-02-28 08:58:08,280] - [data_helper] [140030563579648] - INFO - creating direct client 172.23.105.81:11210 standard_bucket_2 [2018-02-28 08:58:08,611] - [data_helper] [140030563579648] - INFO - creating direct client 172.23.105.86:11210 standard_bucket_2 [2018-02-28 08:58:09,565] - [task] [140030563579648] - INFO - bucket 'standard_bucket_2' was created with per node RAM quota: 757 [2018-02-28 08:58:10,521] - [rest_client] [140030546794240] - INFO - Node versions in cluster [u'5.5.0-2006-enterprise', u'5.5.0-2006-enterprise'] [2018-02-28 08:58:10,522] - [rest_client] [140030546794240] - INFO - http://172.23.105.9:8091/pools/default/buckets with param: bucketType=membase&threadsNumber=3&authType=none&replicaIndex=1&name=standard_bucket_1&evictionPolicy=valueOnly&flushEnabled=1&replicaNumber=1&ramQuotaMB=784 [2018-02-28 08:58:10,580] - [rest_client] [140030546794240] - INFO - 0.06 seconds to create bucket standard_bucket_1 [2018-02-28 08:58:10,729] - [rest_client] [140030546794240] - INFO - Node versions in cluster [u'5.5.0-2006-enterprise', u'5.5.0-2006-enterprise'] [2018-02-28 08:58:10,729] - [rest_client] [140030546794240] - INFO - http://172.23.105.9:8091/pools/default/buckets with param: bucketType=membase&threadsNumber=3&authType=none&replicaIndex=1&name=standard_bucket_2&evictionPolicy=valueOnly&flushEnabled=1&replicaNumber=1&ramQuotaMB=784 [2018-02-28 08:58:11,601] - [rest_client] [140030546794240] - INFO - 0.87 seconds to create bucket standard_bucket_2 [2018-02-28 08:58:11,601] - [bucket_helper] [140030546794240] - INFO - waiting for memcached bucket : standard_bucket_1 in 172.23.105.9 to accept set ops [2018-02-28 08:58:12,036] - [data_helper] [140030546794240] - INFO - creating direct client 172.23.105.9:11210 standard_bucket_1 [2018-02-28 08:58:12,397] - [data_helper] [140030546794240] - INFO - creating direct client 172.23.107.67:11210 standard_bucket_1 [2018-02-28 08:58:12,800] - [data_helper] [140030546794240] - INFO - creating direct client 172.23.105.9:11210 standard_bucket_1 [2018-02-28 08:58:13,155] - [data_helper] [140030546794240] - INFO - creating direct client 172.23.107.67:11210 standard_bucket_1 [2018-02-28 08:58:14,263] - [task] [140030546794240] - INFO - bucket 'standard_bucket_1' was created with per node RAM quota: 784 [2018-02-28 08:58:14,263] - [bucket_helper] [140030546794240] - INFO - waiting for memcached bucket : standard_bucket_2 in 172.23.105.9 to accept set ops [2018-02-28 08:58:14,691] - [data_helper] [140030546794240] - INFO - creating direct client 172.23.105.9:11210 standard_bucket_2 [2018-02-28 08:58:15,040] - [data_helper] [140030546794240] - INFO - creating direct client 172.23.107.67:11210 standard_bucket_2 [2018-02-28 08:58:15,698] - [data_helper] [140030546794240] - INFO - creating direct client 172.23.105.9:11210 standard_bucket_2 [2018-02-28 08:58:16,121] - [data_helper] [140030546794240] - INFO - creating direct client 172.23.107.67:11210 standard_bucket_2 [2018-02-28 08:58:17,043] - [task] [140030546794240] - INFO - bucket 'standard_bucket_2' was created with per node RAM quota: 784 [2018-02-28 08:58:17,044] - [xdcrnewbasetests] [140031333738240] - INFO - ==== XDCRNewbasetests setup is finished for test #10 test_compression_with_unixdcr_incr_load ==== [2018-02-28 08:58:17,091] - [rest_client] [140031333738240] - INFO - adding remote cluster hostname:172.23.105.9:8091 with username:password Administrator:password name:remote_cluster_C1-C2 to source node: 172.23.105.81:8091 [2018-02-28 08:58:17,173] - [rest_client] [140031333738240] - INFO - starting continuous replication type:xmem from standard_bucket_1 to standard_bucket_1 in the remote cluster remote_cluster_C1-C2 with settings {'filterExpression': 'abcd'} [2018-02-28 08:58:17,236] - [rest_client] [140031333738240] - INFO - Replication created with id: 5a2ed9e728f4b857646271101d57641e/standard_bucket_1/standard_bucket_1 [2018-02-28 08:58:27,266] - [rest_client] [140031333738240] - INFO - starting continuous replication type:xmem from standard_bucket_2 to standard_bucket_2 in the remote cluster remote_cluster_C1-C2 with settings {'filterExpression': 'abcd'} [2018-02-28 08:58:27,352] - [rest_client] [140031333738240] - INFO - Replication created with id: 5a2ed9e728f4b857646271101d57641e/standard_bucket_2/standard_bucket_2 [2018-02-28 08:58:37,447] - [rest_client] [140031333738240] - INFO - Updated checkpointInterval=60 on bucket'standard_bucket_1' on 172.23.105.81 [2018-02-28 08:58:37,561] - [rest_client] [140031333738240] - INFO - Updated checkpointInterval=60 on bucket'standard_bucket_2' on 172.23.105.81 [2018-02-28 08:58:37,562] - [xdcrnewbasetests] [140031333738240] - INFO - sleep for 60 secs. ... [2018-02-28 08:59:37,625] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.81 with username:root password:couchbase ssh_key: [2018-02-28 08:59:37,801] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.81 [2018-02-28 08:59:38,272] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.81: curl -X POST -u Administrator:password http://172.23.105.81:8091/settings/replications/5a2ed9e728f4b857646271101d57641e%2Fstandard_bucket_1%2Fstandard_bucket_1 -d compressionType=Snappy [2018-02-28 08:59:38,399] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 08:59:38,400] - [remote_util] [140031333738240] - ERROR - % Total % Received % Xferd Average Speed Time Time Time Current [2018-02-28 08:59:38,401] - [remote_util] [140031333738240] - ERROR - Dload Upload Total Spent Left Speed [2018-02-28 08:59:38,401] - [remote_util] [140031333738240] - ERROR - [2018-02-28 08:59:38,401] - [remote_util] [140031333738240] - ERROR - 0 0 0 0 0 22 0 17973 --:--:-- --:--:-- --:--:-- 17973 [2018-02-28 08:59:38,401] - [remote_util] [140031333738240] - ERROR - 118 333 111 333 0 22 10607 700 --:--:-- --:--:-- --:--:-- 10366 [2018-02-28 08:59:38,402] - [remote_util] [140031333738240] - INFO - {"checkpointInterval":60,"compressionType":"Snappy","docBatchSizeKb":2048,"failureRestartInterval":10,"filterExpression":"abcd","logLevel":"Info","networkUsageLimit":0,"optimisticReplicationThreshold":256,"pauseRequested":false,"sourceNozzlePerNode":2,"statsInterval":1000,"targetNozzlePerNode":2,"type":"xmem","workerBatchSize":500} [2018-02-28 08:59:38,763] - [data_helper] [140031333738240] - INFO - creating direct client 172.23.105.81:11210 standard_bucket_1 [2018-02-28 08:59:39,164] - [data_helper] [140031333738240] - INFO - creating direct client 172.23.105.86:11210 standard_bucket_1 [2018-02-28 08:59:39,757] - [data_helper] [140031333738240] - INFO - creating direct client 172.23.105.81:11210 standard_bucket_2 [2018-02-28 08:59:40,280] - [data_helper] [140031333738240] - INFO - creating direct client 172.23.105.86:11210 standard_bucket_2 [2018-02-28 08:59:52,181] - [xdcrnewbasetests] [140031333738240] - INFO - Updating keys @ C1 [2018-02-28 08:59:52,182] - [xdcrnewbasetests] [140031333738240] - INFO - At bucket 'standard_bucket_1' @ C1: operation: update, key range 0 - 3000 [2018-02-28 08:59:52,399] - [data_helper] [140031333738240] - INFO - creating direct client 172.23.105.81:11210 standard_bucket_1 [2018-02-28 08:59:52,790] - [data_helper] [140031333738240] - INFO - creating direct client 172.23.105.86:11210 standard_bucket_1 [2018-02-28 08:59:53,077] - [xdcrnewbasetests] [140031333738240] - INFO - At bucket 'standard_bucket_2' @ C1: operation: update, key range 0 - 3000 [2018-02-28 08:59:53,339] - [data_helper] [140031333738240] - INFO - creating direct client 172.23.105.81:11210 standard_bucket_2 [2018-02-28 08:59:53,949] - [data_helper] [140031333738240] - INFO - creating direct client 172.23.105.86:11210 standard_bucket_2 [2018-02-28 08:59:56,842] - [xdcrnewbasetests] [140031333738240] - INFO - Deleting keys @ C1 [2018-02-28 08:59:56,844] - [xdcrnewbasetests] [140031333738240] - INFO - At bucket 'standard_bucket_1' @ C1: operation: delete, key range 7000 - 10000 [2018-02-28 08:59:57,187] - [data_helper] [140031333738240] - INFO - creating direct client 172.23.105.81:11210 standard_bucket_1 [2018-02-28 08:59:57,739] - [data_helper] [140031333738240] - INFO - creating direct client 172.23.105.86:11210 standard_bucket_1 [2018-02-28 08:59:58,114] - [xdcrnewbasetests] [140031333738240] - INFO - At bucket 'standard_bucket_2' @ C1: operation: delete, key range 7000 - 10000 [2018-02-28 08:59:58,369] - [data_helper] [140031333738240] - INFO - creating direct client 172.23.105.81:11210 standard_bucket_2 [2018-02-28 08:59:58,777] - [data_helper] [140031333738240] - INFO - creating direct client 172.23.105.86:11210 standard_bucket_2 [2018-02-28 09:00:02,304] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.81 with username:root password:couchbase ssh_key: [2018-02-28 09:00:02,470] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.81 [2018-02-28 09:00:02,929] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.81: curl -u Administrator:password http://172.23.105.81:8091/settings/replications/5a2ed9e728f4b857646271101d57641e%2Fstandard_bucket_1%2Fstandard_bucket_1 [2018-02-28 09:00:03,049] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:00:03,050] - [remote_util] [140031333738240] - ERROR - % Total % Received % Xferd Average Speed Time Time Time Current [2018-02-28 09:00:03,051] - [remote_util] [140031333738240] - ERROR - Dload Upload Total Spent Left Speed [2018-02-28 09:00:03,051] - [remote_util] [140031333738240] - ERROR - [2018-02-28 09:00:03,051] - [remote_util] [140031333738240] - ERROR - 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 [2018-02-28 09:00:03,051] - [remote_util] [140031333738240] - ERROR - 111 333 111 333 0 0 38865 0 --:--:-- --:--:-- --:--:-- 47571 [2018-02-28 09:00:03,052] - [remote_util] [140031333738240] - INFO - {"checkpointInterval":60,"compressionType":"Snappy","docBatchSizeKb":2048,"failureRestartInterval":10,"filterExpression":"abcd","logLevel":"Info","networkUsageLimit":0,"optimisticReplicationThreshold":256,"pauseRequested":false,"sourceNozzlePerNode":2,"statsInterval":1000,"targetNozzlePerNode":2,"type":"xmem","workerBatchSize":500} [2018-02-28 09:00:03,052] - [compressionXDCR] [140031333738240] - INFO - Compression Type for replication 5a2ed9e728f4b857646271101d57641e%2Fstandard_bucket_1%2Fstandard_bucket_1 is Snappy [2018-02-28 09:00:03,052] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.81: curl -u Administrator:password http://172.23.105.81:8091/pools/default/buckets/standard_bucket_1/stats/replications%2F5a2ed9e728f4b857646271101d57641e%2Fstandard_bucket_1%2Fstandard_bucket_1%2Fdata_replicated [2018-02-28 09:00:03,167] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:00:03,168] - [remote_util] [140031333738240] - ERROR - % Total % Received % Xferd Average Speed Time Time Time Current [2018-02-28 09:00:03,168] - [remote_util] [140031333738240] - ERROR - Dload Upload Total Spent Left Speed [2018-02-28 09:00:03,168] - [remote_util] [140031333738240] - ERROR - [2018-02-28 09:00:03,168] - [remote_util] [140031333738240] - ERROR - 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 [2018-02-28 09:00:03,169] - [remote_util] [140031333738240] - ERROR - 101 1218 101 1218 0 0 159k 0 --:--:-- --:--:-- --:--:-- 169k [2018-02-28 09:00:03,169] - [remote_util] [140031333738240] - INFO - {"samplesCount":60,"isPersistent":true,"lastTStamp":1519837202071,"interval":1000,"timestamp":[1519837144071,1519837145071,1519837146071,1519837147071,1519837148071,1519837149071,1519837150071,1519837151071,1519837152071,1519837153071,1519837154071,1519837155071,1519837156071,1519837157071,1519837158071,1519837159071,1519837160071,1519837161071,1519837162071,1519837163071,1519837164071,1519837165071,1519837166071,1519837167071,1519837168071,1519837169071,1519837170071,1519837171071,1519837172071,1519837173071,1519837174071,1519837175071,1519837176071,1519837177071,1519837178071,1519837179071,1519837180071,1519837181072,1519837182071,1519837183071,1519837184071,1519837185071,1519837186071,1519837187071,1519837188071,1519837189071,1519837190071,1519837191071,1519837192071,1519837193071,1519837194071,1519837195072,1519837196071,1519837197071,1519837198071,1519837199071,1519837200071,1519837201072,1519837202071],"nodeStats":{"172.23.105.81:8091":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"172.23.105.86:8091":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}} [2018-02-28 09:00:03,169] - [compressionXDCR] [140031333738240] - INFO - Compressed data for replication 5a2ed9e728f4b857646271101d57641e%2Fstandard_bucket_1%2Fstandard_bucket_1 is 0 [2018-02-28 09:00:03,170] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.81: curl -u Administrator:password http://172.23.105.81:8091/pools/default/buckets/standard_bucket_2/stats/replications%2F5a2ed9e728f4b857646271101d57641e%2Fstandard_bucket_2%2Fstandard_bucket_2%2Fdata_replicated [2018-02-28 09:00:03,286] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:00:03,287] - [remote_util] [140031333738240] - ERROR - % Total % Received % Xferd Average Speed Time Time Time Current [2018-02-28 09:00:03,287] - [remote_util] [140031333738240] - ERROR - Dload Upload Total Spent Left Speed [2018-02-28 09:00:03,287] - [remote_util] [140031333738240] - ERROR - [2018-02-28 09:00:03,288] - [remote_util] [140031333738240] - ERROR - 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 [2018-02-28 09:00:03,288] - [remote_util] [140031333738240] - ERROR - 101 1218 101 1218 0 0 159k 0 --:--:-- --:--:-- --:--:-- 237k [2018-02-28 09:00:03,288] - [remote_util] [140031333738240] - INFO - {"samplesCount":60,"isPersistent":true,"lastTStamp":1519837202071,"interval":1000,"timestamp":[1519837144071,1519837145071,1519837146071,1519837147071,1519837148071,1519837149071,1519837150071,1519837151071,1519837152071,1519837153071,1519837154071,1519837155071,1519837156071,1519837157071,1519837158071,1519837159071,1519837160071,1519837161071,1519837162071,1519837163071,1519837164071,1519837165071,1519837166071,1519837167071,1519837168071,1519837169071,1519837170071,1519837171071,1519837172071,1519837173071,1519837174071,1519837175071,1519837176071,1519837177071,1519837178071,1519837179071,1519837180071,1519837181072,1519837182071,1519837183071,1519837184071,1519837185071,1519837186071,1519837187071,1519837188071,1519837189071,1519837190071,1519837191071,1519837192071,1519837193071,1519837194071,1519837195072,1519837196071,1519837197071,1519837198071,1519837199071,1519837200071,1519837201072,1519837202071],"nodeStats":{"172.23.105.81:8091":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"172.23.105.86:8091":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}} [2018-02-28 09:00:03,288] - [compressionXDCR] [140031333738240] - INFO - Uncompressed data for replication 5a2ed9e728f4b857646271101d57641e%2Fstandard_bucket_2%2Fstandard_bucket_2 is 0 [2018-02-28 09:00:03,289] - [xdcrnewbasetests] [140031333738240] - INFO - Collecting logs @ 172.23.105.81 [2018-02-28 09:00:03,292] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.81 with username:root password:couchbase ssh_key: [2018-02-28 09:00:03,464] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.81 [2018-02-28 09:00:03,870] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.81: /opt/couchbase/bin/cbcollect_info 172.23.105.81-20180228-0900-diag.zip [2018-02-28 09:02:39,925] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:02:40,843] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.81: rm -f /root/172.23.105.81-20180228-0900-diag.zip [2018-02-28 09:02:40,864] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:02:40,869] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.81 with username:root password:couchbase ssh_key: [2018-02-28 09:02:41,057] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.81 [2018-02-28 09:02:41,620] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.81: tar -zcvf 172.23.105.81-2282018-92-couch.tar.gz '/opt/couchbase/var/lib/couchbase/data' >/dev/null 2>&1 [2018-02-28 09:02:43,177] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:02:43,637] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.81: rm -f /root/172.23.105.81-2282018-92-couch.tar.gz [2018-02-28 09:02:43,658] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:02:43,660] - [xdcrnewbasetests] [140031333738240] - INFO - Collecting logs @ 172.23.105.86 [2018-02-28 09:02:43,664] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.86 with username:root password:couchbase ssh_key: [2018-02-28 09:02:43,872] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.86 [2018-02-28 09:02:44,210] - [remote_util] [140031333738240] - INFO - It could be other operating system. Go to check at other location [2018-02-28 09:02:44,633] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.86: /opt/couchbase/bin/cbcollect_info 172.23.105.86-20180228-0902-diag.zip [2018-02-28 09:06:27,617] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:06:28,428] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.86: rm -f /root/172.23.105.86-20180228-0902-diag.zip [2018-02-28 09:06:28,458] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:06:28,465] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.86 with username:root password:couchbase ssh_key: [2018-02-28 09:06:28,785] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.86 [2018-02-28 09:06:29,105] - [remote_util] [140031333738240] - INFO - It could be other operating system. Go to check at other location [2018-02-28 09:06:29,571] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.86: tar -zcvf 172.23.105.86-2282018-96-couch.tar.gz '/opt/couchbase/var/lib/couchbase/data' >/dev/null 2>&1 [2018-02-28 09:06:31,994] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:06:32,395] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.86: rm -f /root/172.23.105.86-2282018-96-couch.tar.gz [2018-02-28 09:06:32,420] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:06:32,422] - [xdcrnewbasetests] [140031333738240] - INFO - Collecting logs @ 172.23.105.9 [2018-02-28 09:06:32,425] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.9 with username:root password:couchbase ssh_key: [2018-02-28 09:06:32,605] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.9 [2018-02-28 09:06:33,016] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.9: /opt/couchbase/bin/cbcollect_info 172.23.105.9-20180228-0906-diag.zip [2018-02-28 09:09:04,319] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:09:05,003] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.9: rm -f /root/172.23.105.9-20180228-0906-diag.zip [2018-02-28 09:09:05,086] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:09:05,093] - [remote_util] [140031333738240] - INFO - connecting to 172.23.105.9 with username:root password:couchbase ssh_key: [2018-02-28 09:09:05,313] - [remote_util] [140031333738240] - INFO - Connected to 172.23.105.9 [2018-02-28 09:09:06,445] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.9: tar -zcvf 172.23.105.9-2282018-99-couch.tar.gz '/opt/couchbase/var/lib/couchbase/data' >/dev/null 2>&1 [2018-02-28 09:09:06,714] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:09:07,493] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.105.9: rm -f /root/172.23.105.9-2282018-99-couch.tar.gz [2018-02-28 09:09:07,512] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:09:07,515] - [xdcrnewbasetests] [140031333738240] - INFO - Collecting logs @ 172.23.107.67 [2018-02-28 09:09:07,518] - [remote_util] [140031333738240] - INFO - connecting to 172.23.107.67 with username:root password:couchbase ssh_key: [2018-02-28 09:09:07,685] - [remote_util] [140031333738240] - INFO - Connected to 172.23.107.67 [2018-02-28 09:09:08,128] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.67: /opt/couchbase/bin/cbcollect_info 172.23.107.67-20180228-0909-diag.zip [2018-02-28 09:11:22,836] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:11:23,387] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.67: rm -f /root/172.23.107.67-20180228-0909-diag.zip [2018-02-28 09:11:23,472] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:11:23,593] - [remote_util] [140031333738240] - INFO - connecting to 172.23.107.67 with username:root password:couchbase ssh_key: [2018-02-28 09:11:23,738] - [remote_util] [140031333738240] - INFO - Connected to 172.23.107.67 [2018-02-28 09:11:24,231] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.67: tar -zcvf 172.23.107.67-2282018-911-couch.tar.gz '/opt/couchbase/var/lib/couchbase/data' >/dev/null 2>&1 [2018-02-28 09:11:24,501] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:11:24,708] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.67: rm -f /root/172.23.107.67-2282018-911-couch.tar.gz [2018-02-28 09:11:24,785] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:11:24,787] - [xdcrnewbasetests] [140031333738240] - INFO - Collecting logs @ 172.23.107.78 [2018-02-28 09:11:24,790] - [remote_util] [140031333738240] - INFO - connecting to 172.23.107.78 with username:root password:couchbase ssh_key: [2018-02-28 09:11:24,995] - [remote_util] [140031333738240] - INFO - Connected to 172.23.107.78 [2018-02-28 09:11:25,421] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.78: /opt/couchbase/bin/cbcollect_info 172.23.107.78-20180228-0911-diag.zip [2018-02-28 09:12:35,312] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:12:35,563] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.78: rm -f /root/172.23.107.78-20180228-0911-diag.zip [2018-02-28 09:12:35,580] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:12:35,697] - [remote_util] [140031333738240] - INFO - connecting to 172.23.107.78 with username:root password:couchbase ssh_key: [2018-02-28 09:12:35,832] - [remote_util] [140031333738240] - INFO - Connected to 172.23.107.78 [2018-02-28 09:12:36,317] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.78: tar -zcvf 172.23.107.78-2282018-912-couch.tar.gz '/opt/couchbase/var/lib/couchbase/data' >/dev/null 2>&1 [2018-02-28 09:12:36,336] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:12:36,469] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.107.78: rm -f /root/172.23.107.78-2282018-912-couch.tar.gz [2018-02-28 09:12:36,482] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:12:36,484] - [xdcrnewbasetests] [140031333738240] - INFO - Collecting logs @ 172.23.108.109 [2018-02-28 09:12:36,486] - [remote_util] [140031333738240] - INFO - connecting to 172.23.108.109 with username:root password:couchbase ssh_key: [2018-02-28 09:12:36,688] - [remote_util] [140031333738240] - INFO - Connected to 172.23.108.109 [2018-02-28 09:12:37,289] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.108.109: /opt/couchbase/bin/cbcollect_info 172.23.108.109-20180228-0912-diag.zip [2018-02-28 09:14:22,293] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:14:22,572] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.108.109: rm -f /root/172.23.108.109-20180228-0912-diag.zip [2018-02-28 09:14:22,617] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:14:22,624] - [remote_util] [140031333738240] - INFO - connecting to 172.23.108.109 with username:root password:couchbase ssh_key: [2018-02-28 09:14:22,824] - [remote_util] [140031333738240] - INFO - Connected to 172.23.108.109 [2018-02-28 09:14:23,553] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.108.109: tar -zcvf 172.23.108.109-2282018-914-couch.tar.gz '/opt/couchbase/var/lib/couchbase/data' >/dev/null 2>&1 [2018-02-28 09:14:23,593] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:14:23,810] - [remote_util] [140031333738240] - INFO - running command.raw on 172.23.108.109: rm -f /root/172.23.108.109-2282018-914-couch.tar.gz [2018-02-28 09:14:23,843] - [remote_util] [140031333738240] - INFO - command executed successfully [2018-02-28 09:14:24,208] - [xdcrnewbasetests] [140031333738240] - INFO - ==== XDCRNewbasetests cleanup is started for test #10 test_compression_with_unixdcr_incr_load ==== [2018-02-28 09:14:24,208] - [xdcrnewbasetests] [140031333738240] - INFO - removing xdcr/nodes settings [2018-02-28 09:14:24,314] - [rest_client] [140031333738240] - INFO - Deleting replication /controller/cancelXDCR/5a2ed9e728f4b857646271101d57641e%2Fstandard_bucket_1%2Fstandard_bucket_1 [2018-02-28 09:14:24,363] - [rest_client] [140031333738240] - INFO - Replication deleted successfully [2018-02-28 09:14:24,364] - [rest_client] [140031333738240] - INFO - Deleting replication /controller/cancelXDCR/5a2ed9e728f4b857646271101d57641e%2Fstandard_bucket_2%2Fstandard_bucket_2 [2018-02-28 09:14:24,395] - [rest_client] [140031333738240] - INFO - Replication deleted successfully [2018-02-28 09:14:24,427] - [rest_client] [140031333738240] - INFO - removing remote cluster name:remote_cluster_C1-C2 [2018-02-28 09:14:24,571] - [xdcrnewbasetests] [140031333738240] - INFO - cleanup [ip:172.23.105.81 port:8091 ssh_username:root, ip:172.23.105.86 port:8091 ssh_username:root] [2018-02-28 09:14:24,674] - [bucket_helper] [140031333738240] - INFO - deleting existing buckets [u'standard_bucket_1', u'standard_bucket_2'] on 172.23.105.81 [2018-02-28 09:14:24,675] - [bucket_helper] [140031333738240] - INFO - remove bucket standard_bucket_1 ... [2018-02-28 09:14:25,729] - [bucket_helper] [140031333738240] - INFO - deleted bucket : standard_bucket_1 from 172.23.105.81 [2018-02-28 09:14:25,730] - [bucket_helper] [140031333738240] - INFO - waiting for bucket deletion to complete.... [2018-02-28 09:14:25,794] - [rest_client] [140031333738240] - INFO - node 172.23.105.81 existing buckets : [u'standard_bucket_2'] [2018-02-28 09:14:25,794] - [bucket_helper] [140031333738240] - INFO - remove bucket standard_bucket_2 ... [2018-02-28 09:14:26,719] - [bucket_helper] [140031333738240] - INFO - deleted bucket : standard_bucket_2 from 172.23.105.81 [2018-02-28 09:14:26,720] - [bucket_helper] [140031333738240] - INFO - waiting for bucket deletion to complete.... [2018-02-28 09:14:26,740] - [rest_client] [140031333738240] - INFO - node 172.23.105.81 existing buckets : [] [2018-02-28 09:14:26,740] - [bucket_helper] [140031333738240] - INFO - sleep 2 seconds to make sure all buckets were deleted completely. [2018-02-28 09:14:28,894] - [cluster_helper] [140031333738240] - INFO - rebalancing all nodes in order to remove nodes [2018-02-28 09:14:28,953] - [rest_client] [140031333738240] - INFO - rebalance params : {'password': 'password', 'ejectedNodes': u'ns_1@172.23.105.86', 'user': 'Administrator', 'knownNodes': u'ns_1@172.23.105.81,ns_1@172.23.105.86'} [2018-02-28 09:14:28,991] - [rest_client] [140031333738240] - INFO - rebalance operation started [2018-02-28 09:14:29,030] - [rest_client] [140031333738240] - INFO - rebalance percentage : 100.00 % [2018-02-28 09:14:39,084] - [rest_client] [140031333738240] - INFO - rebalance progress took 10.0930931568 seconds [2018-02-28 09:14:39,085] - [rest_client] [140031333738240] - INFO - sleep for 10 seconds after rebalance... [2018-02-28 09:14:49,184] - [cluster_helper] [140031333738240] - INFO - removed all the nodes from cluster associated with ip:172.23.105.81 port:8091 ssh_username:root ? [(u'ns_1@172.23.105.86', 8091)] [2018-02-28 09:14:49,227] - [cluster_helper] [140031333738240] - INFO - waiting for ns_server @ 172.23.105.81:8091 [2018-02-28 09:14:49,268] - [cluster_helper] [140031333738240] - INFO - ns_server @ 172.23.105.81:8091 is running [2018-02-28 09:14:49,396] - [bucket_helper] [140031333738240] - INFO - deleting existing buckets [] on 172.23.105.86 [2018-02-28 09:14:49,397] - [bucket_helper] [140031333738240] - INFO - sleep 2 seconds to make sure all buckets were deleted completely. [2018-02-28 09:14:51,591] - [cluster_helper] [140031333738240] - INFO - waiting for ns_server @ 172.23.105.86:8091 [2018-02-28 09:14:51,625] - [cluster_helper] [140031333738240] - INFO - ns_server @ 172.23.105.86:8091 is running [2018-02-28 09:14:51,625] - [xdcrnewbasetests] [140031333738240] - INFO - removing xdcr/nodes settings [2018-02-28 09:14:51,840] - [xdcrnewbasetests] [140031333738240] - INFO - cleanup [ip:172.23.105.9 port:8091 ssh_username:root, ip:172.23.107.67 port:8091 ssh_username:root] [2018-02-28 09:14:51,989] - [bucket_helper] [140031333738240] - INFO - deleting existing buckets [u'standard_bucket_1', u'standard_bucket_2'] on 172.23.105.9 [2018-02-28 09:14:51,989] - [bucket_helper] [140031333738240] - INFO - remove bucket standard_bucket_1 ... [2018-02-28 09:14:53,207] - [bucket_helper] [140031333738240] - INFO - deleted bucket : standard_bucket_1 from 172.23.105.9 [2018-02-28 09:14:53,207] - [bucket_helper] [140031333738240] - INFO - waiting for bucket deletion to complete.... [2018-02-28 09:14:53,245] - [rest_client] [140031333738240] - INFO - node 172.23.105.9 existing buckets : [u'standard_bucket_2'] [2018-02-28 09:14:53,246] - [bucket_helper] [140031333738240] - INFO - remove bucket standard_bucket_2 ... [2018-02-28 09:14:54,090] - [bucket_helper] [140031333738240] - INFO - deleted bucket : standard_bucket_2 from 172.23.105.9 [2018-02-28 09:14:54,091] - [bucket_helper] [140031333738240] - INFO - waiting for bucket deletion to complete.... [2018-02-28 09:14:54,126] - [rest_client] [140031333738240] - INFO - node 172.23.105.9 existing buckets : [] [2018-02-28 09:14:54,126] - [bucket_helper] [140031333738240] - INFO - sleep 2 seconds to make sure all buckets were deleted completely. [2018-02-28 09:14:56,278] - [cluster_helper] [140031333738240] - INFO - rebalancing all nodes in order to remove nodes [2018-02-28 09:14:56,317] - [rest_client] [140031333738240] - INFO - rebalance params : {'password': 'password', 'ejectedNodes': u'ns_1@172.23.107.67', 'user': 'Administrator', 'knownNodes': u'ns_1@172.23.105.9,ns_1@172.23.107.67'} [2018-02-28 09:14:56,360] - [rest_client] [140031333738240] - INFO - rebalance operation started [2018-02-28 09:15:06,460] - [rest_client] [140031333738240] - INFO - rebalance progress took 10.0995030403 seconds [2018-02-28 09:15:06,460] - [rest_client] [140031333738240] - INFO - sleep for 10 seconds after rebalance... [2018-02-28 09:15:16,549] - [cluster_helper] [140031333738240] - INFO - removed all the nodes from cluster associated with ip:172.23.105.9 port:8091 ssh_username:root ? [(u'ns_1@172.23.107.67', 8091)] [2018-02-28 09:15:16,590] - [cluster_helper] [140031333738240] - INFO - waiting for ns_server @ 172.23.105.9:8091 [2018-02-28 09:15:16,628] - [cluster_helper] [140031333738240] - INFO - ns_server @ 172.23.105.9:8091 is running [2018-02-28 09:15:16,705] - [bucket_helper] [140031333738240] - INFO - deleting existing buckets [] on 172.23.107.67 [2018-02-28 09:15:16,705] - [bucket_helper] [140031333738240] - INFO - sleep 2 seconds to make sure all buckets were deleted completely. [2018-02-28 09:15:18,888] - [cluster_helper] [140031333738240] - INFO - waiting for ns_server @ 172.23.107.67:8091 [2018-02-28 09:15:18,922] - [cluster_helper] [140031333738240] - INFO - ns_server @ 172.23.107.67:8091 is running [2018-02-28 09:15:18,922] - [xdcrnewbasetests] [140031333738240] - INFO - ==== XDCRNewbasetests cleanup is finished for test #10 test_compression_with_unixdcr_incr_load ===